@fin.cx/opendata 3.5.0 → 3.6.0

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@fin.cx/opendata',
-  version: '3.5.0',
+  version: '3.6.0',
   description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
 }

package/ts/classes.businessrecord.ts

@@ -18,7 +18,7 @@ export class BusinessRecord extends plugins.smartdata.SmartDataDbDoc<
 
   // INSTANCE
   @plugins.smartdata.unI()
-  id: string;
+  id!: string;
 
   @plugins.smartdata.svDb()
   data: {

package/ts/classes.handelsregister.ts

@@ -20,15 +20,44 @@ export class HandelsRegister {
     this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple());
   }
 
+  private async resetDownloadFolder() {
+    await plugins.smartfs.directory(this.uniqueDowloadFolder).recursive().delete().catch(() => {});
+    await plugins.smartfs.directory(this.uniqueDowloadFolder).create();
+  }
+
+  private async waitForDownloadedFile() {
+    for (let attempt = 0; attempt < 120; attempt++) {
+      const directoryEntries = await plugins.fs.readdir(this.uniqueDowloadFolder);
+      const fileName = directoryEntries.find(
+        (entry) => !entry.endsWith('.crdownload') && !entry.endsWith('.tmp')
+      );
+
+      if (fileName) {
+        const filePath = plugins.path.join(this.uniqueDowloadFolder, fileName);
+        const firstStat = await plugins.fs.stat(filePath);
+        await plugins.smartdelay.delayFor(500);
+        const secondStat = await plugins.fs.stat(filePath);
+
+        if (firstStat.size === secondStat.size) {
+          return filePath;
+        }
+      }
+
+      await plugins.smartdelay.delayFor(500);
+    }
+
+    throw new Error('Timed out while waiting for the download to finish.');
+  }
+
   public async start() {
     // Start the browser
-    await plugins.smartfile.fs.ensureDir(this.uniqueDowloadFolder);
+    await this.resetDownloadFolder();
     await this.smartbrowserInstance.start();
   }
 
   public async stop() {
     // Stop the browser
-    await plugins.smartfile.fs.remove(this.uniqueDowloadFolder);
+    await plugins.smartfs.directory(this.uniqueDowloadFolder).recursive().delete();
     await this.smartbrowserInstance.stop();
   }
 
@@ -184,24 +213,16 @@ export class HandelsRegister {
           throw new Error('Invalid file type');
       }
     }, typeArg);
-
-
-    await plugins.smartfile.fs.waitForFileToBeReady(this.uniqueDowloadFolder);
-
-    const files = await plugins.smartfile.fs.fileTreeToObject(this.uniqueDowloadFolder, '**/*');
-    const file = files[0];
-
-    // lets clear the folder for the next download
-    await plugins.smartfile.fs.ensureEmptyDir(this.uniqueDowloadFolder);
-    switch (typeArg) {
-      case 'AD':
-        await file.rename(`ad.pdf`);
-        break;
-      case 'SI':
-        await file.rename(`si.xml`);
-        break;
-        break;
-    }
+    const downloadedFilePath = await this.waitForDownloadedFile();
+    const renamedFilePath = plugins.path.join(
+      this.uniqueDowloadFolder,
+      typeArg === 'AD' ? 'ad.pdf' : 'si.xml'
+    );
+    await plugins.fs.rename(downloadedFilePath, renamedFilePath);
+    const file = await plugins.smartfileFactory.fromFilePath(renamedFilePath);
+
+    // Keep the download folder empty between requests.
+    await this.resetDownloadFolder();
     return file;
   }
 
@@ -297,6 +318,10 @@ export class HandelsRegister {
    */
   public async getSpecificCompany(companyArg: BusinessRecord['data']['germanParsedRegistration']) {
     return this.asyncExecutionStack.getExclusiveExecutionSlot(async () => {
+      if (!companyArg?.type || !companyArg.number || !companyArg.court) {
+        throw new Error('A complete parsed German registration is required.');
+      }
+
       const page = await this.getNewPage();
       await this.navigateToPage(page, 'Normal search');
       await page.waitForSelector('#form\\:schlagwoerter', { timeout: 5000 });

package/ts/classes.jsonldata.ts

@@ -53,14 +53,15 @@ export class JsonlDataProcessor<T> {
     dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
   ) {
     const done = plugins.smartpromise.defer();
-    const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir);
+    const dataExists = await plugins.smartfs.directory(this.germanBusinessDataDir).exists();
     if (!dataExists) {
-      await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir);
+      await plugins.smartfs.directory(this.germanBusinessDataDir).create();
     } else {
     }
 
-    const smartarchive = await plugins.smartarchive.SmartArchive.fromArchiveUrl(dataUrlArg);
-    const jsonlDataStream = await smartarchive.exportToStreamOfStreamFiles();
+    const jsonlDataStream = await plugins.smartarchive.SmartArchive.create()
+      .url(dataUrlArg)
+      .toStreamFiles();
     let totalRecordsCounter = 0;
     let nextRest: string = '';
     jsonlDataStream.pipe(
@@ -74,11 +75,11 @@ export class JsonlDataProcessor<T> {
               writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
                 const currentString = nextRest + chunkArg.toString();
                 const lines = currentString.split('\n');
-                nextRest = lines.pop();
+                nextRest = lines.pop() ?? '';
                 console.log(`Got another ${lines.length} records.`);
                 const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor<string>(
                   async (line) => {
-                    let entry: T;
+                    let entry: T | undefined;
                     if (!line) return;
                     try {
                       entry = JSON.parse(line);

package/ts/classes.main.opendata.ts

@@ -1,7 +1,6 @@
 import { BusinessRecord } from './classes.businessrecord.js';
 import { HandelsRegister } from './classes.handelsregister.js';
 import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
-import * as paths from './paths.js';
 import * as plugins from './plugins.js';
 
 export interface IOpenDataConfig {
@@ -11,12 +10,13 @@ export interface IOpenDataConfig {
 }
 
 export class OpenData {
-  public db: plugins.smartdata.SmartdataDb;
-  private serviceQenv: plugins.qenv.Qenv;
+  public db!: plugins.smartdata.SmartdataDb;
+  private localSmartDb?: plugins.smartdb.LocalSmartDb;
   private config: IOpenDataConfig;
+  private started = false;
 
-  public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
-  public handelsregister: HandelsRegister;
+  public jsonLDataProcessor!: JsonlDataProcessor<SeedEntryType>;
+  public handelsregister!: HandelsRegister;
 
   public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
 
@@ -28,38 +28,62 @@ export class OpenData {
       throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
     }
     this.config = configArg;
-    this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir);
   }
 
   public async start() {
+    if (this.started) {
+      return;
+    }
+
     // Ensure configured directories exist
-    await plugins.smartfile.fs.ensureDir(this.config.nogitDir);
-    await plugins.smartfile.fs.ensureDir(this.config.downloadDir);
-    await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir);
+    await plugins.smartfs.directory(this.config.nogitDir).create();
+    await plugins.smartfs.directory(this.config.downloadDir).create();
+    await plugins.smartfs.directory(this.config.germanBusinessDataDir).create();
 
+    this.localSmartDb = new plugins.smartdb.LocalSmartDb({
+      folderPath: plugins.path.join(this.config.nogitDir, 'opendata-smartdb'),
+    });
+
+    const connectionInfo = await this.localSmartDb.start();
     this.db = new plugins.smartdata.SmartdataDb({
-      mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
-      mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
-      mongoDbUser: await this.serviceQenv.getEnvVarOnDemand('MONGODB_USER'),
-      mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
+      mongoDbUrl: connectionInfo.connectionUri,
+      mongoDbName: 'opendata',
     });
-    await this.db.init();
-    this.jsonLDataProcessor = new JsonlDataProcessor(
-      this.config.germanBusinessDataDir,
-      async (entryArg) => {
-        const businessRecord = new this.CBusinessRecord();
-        businessRecord.id = await this.CBusinessRecord.getNewId();
-        businessRecord.data.name = entryArg.name;
-        businessRecord.data.germanParsedRegistration = {
-          court: entryArg.all_attributes.registered_office,
-          number: entryArg.all_attributes._registerNummer,
-          type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
-        };
-        await businessRecord.save();
+
+    try {
+      await this.db.init();
+      await this.db.mongoDb.collection('_opendata_bootstrap').insertOne({
+        createdAt: new Date(),
+      });
+      await this.db.mongoDb.collection('_opendata_bootstrap').deleteMany({});
+
+      this.jsonLDataProcessor = new JsonlDataProcessor(
+        this.config.germanBusinessDataDir,
+        async (entryArg) => {
+          const businessRecord = new this.CBusinessRecord();
+          businessRecord.id = await this.CBusinessRecord.getNewId();
+          businessRecord.data.name = entryArg.name;
+          businessRecord.data.germanParsedRegistration = {
+            court: entryArg.all_attributes.registered_office,
+            number: entryArg.all_attributes._registerNummer,
+            type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
+          };
+          await businessRecord.save();
+        }
+      );
+
+      this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
+      await this.handelsregister.start();
+      this.started = true;
+    } catch (error) {
+      if (this.handelsregister) {
+        await this.handelsregister.stop().catch(() => {});
       }
-    );
-    this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
-    await this.handelsregister.start();
+      await this.db.close().catch(() => {});
+      await this.localSmartDb.stop().catch(() => {});
+      this.localSmartDb = undefined;
+      throw error;
+    }
   }
 
   public async buildInitialDb() {
@@ -81,7 +105,16 @@ export class OpenData {
 
 
   public async stop() {
+    if (!this.started) {
+      return;
+    }
+
+    if (this.handelsregister) {
+      await this.handelsregister.stop();
+    }
     await this.db.close();
-    await this.handelsregister.stop();
+    await this.localSmartDb?.stop();
+    this.localSmartDb = undefined;
+    this.started = false;
   }
 }

package/ts/index.ts

@@ -1,2 +1,3 @@
 export * from './classes.main.opendata.js';
+export * from './laws/index.js';
 export * from './stocks/index.js';

package/ts/laws/classes.lawrecord.ts

@@ -0,0 +1,81 @@
+import * as plugins from '../plugins.js';
+import type { TJurisdiction, TLawSource, TRawLawFormat } from './interfaces.law.js';
+
+@plugins.smartdata.Manager()
+export class LawRecord extends plugins.smartdata.SmartDataDbDoc<LawRecord, LawRecord> {
+  public static getByLookupKey = async (lookupKeyArg: string) => {
+    const lawRecords = await LawRecord.getInstances({
+      lookupKey: lookupKeyArg,
+    });
+    return lawRecords[0];
+  };
+
+  @plugins.smartdata.unI()
+  id!: string;
+
+  @plugins.smartdata.unI()
+  @plugins.smartdata.svDb()
+  lookupKey!: string;
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  jurisdiction!: TJurisdiction;
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  source!: TLawSource;
+
+  @plugins.smartdata.searchable()
+  @plugins.smartdata.svDb()
+  identifier!: string;
+
+  @plugins.smartdata.searchable()
+  @plugins.smartdata.svDb()
+  title!: string;
+
+  @plugins.smartdata.searchable()
+  @plugins.smartdata.svDb()
+  shortTitle: string = '';
+
+  @plugins.smartdata.searchable()
+  @plugins.smartdata.svDb()
+  citation: string = '';
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  type: string = '';
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  language: string = '';
+
+  @plugins.smartdata.svDb()
+  sourceUrl!: string;
+
+  @plugins.smartdata.svDb()
+  rawFormat!: TRawLawFormat;
+
+  @plugins.smartdata.svDb()
+  rawBody!: string;
+
+  @plugins.smartdata.searchable()
+  @plugins.smartdata.svDb()
+  text!: string;
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  dateIssued: string = '';
+
+  @plugins.smartdata.index()
+  @plugins.smartdata.svDb()
+  lastModified: string = '';
+
+  @plugins.smartdata.svDb()
+  sourceMeta: Record<string, string> = {};
+
+  @plugins.smartdata.svDb()
+  fetchedAt: Date = new Date();
+
+  @plugins.smartdata.svDb()
+  syncedAt: Date = new Date();
+}