@fin.cx/opendata 3.6.0 → 4.0.0

package/ts/classes.jsonldata.ts

@@ -1,114 +0,0 @@
-import * as plugins from './plugins.js';
-import type { OpenData } from './classes.main.opendata.js';
-
-export type SeedEntryType = {
-  all_attributes: {
-    _registerArt: string;
-    _registerNummer: string;
-    additional_data: {
-      AD: boolean;
-      CD: boolean;
-      DK: boolean;
-      HD: boolean;
-      SI: boolean;
-      UT: boolean;
-      VÖ: boolean;
-    };
-    federal_state: string;
-    native_company_number: string;
-    registered_office: string;
-    registrar: string;
-  };
-  company_number: string;
-  current_status: string;
-  jurisdiction_code: string;
-  name: string;
-  officers: {
-    name: string;
-    other_attributes: {
-      city: string;
-      firstname: string;
-      flag: string;
-      lastname: string;
-    };
-    position: string;
-    start_date: string; // ISO 8601 date string
-    type: string;
-  }[];
-  registered_address: string;
-  retrieved_at: string; // ISO 8601 date string
-};
-
-export class JsonlDataProcessor<T> {
-  private germanBusinessDataDir: string;
-  public forEachFunction: (entryArg: T) => Promise<void>;
-
-  constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) {
-    this.germanBusinessDataDir = germanBusinessDataDirArg;
-    this.forEachFunction = forEachFunctionArg;
-  }
-
-  // TODO: define a mapper as argument instead of hard-coding it
-  public async processDataFromUrl(
-    dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
-  ) {
-    const done = plugins.smartpromise.defer();
-    const dataExists = await plugins.smartfs.directory(this.germanBusinessDataDir).exists();
-    if (!dataExists) {
-      await plugins.smartfs.directory(this.germanBusinessDataDir).create();
-    } else {
-    }
-
-    const jsonlDataStream = await plugins.smartarchive.SmartArchive.create()
-      .url(dataUrlArg)
-      .toStreamFiles();
-    let totalRecordsCounter = 0;
-    let nextRest: string = '';
-    jsonlDataStream.pipe(
-      new plugins.smartstream.SmartDuplex({
-        objectMode: true,
-        writeFunction: async (chunkArg: plugins.smartfile.StreamFile, streamToolsArg) => {
-          const readStream = await chunkArg.createReadStream();
-          readStream.pipe(
-            new plugins.smartstream.SmartDuplex({
-              objectMode: true,
-              writeFunction: async (chunkArg: Buffer, streamToolsArg) => {
-                const currentString = nextRest + chunkArg.toString();
-                const lines = currentString.split('\n');
-                nextRest = lines.pop() ?? '';
-                console.log(`Got another ${lines.length} records.`);
-                const concurrentProcessor = new plugins.smartarray.ConcurrentProcessor<string>(
-                  async (line) => {
-                    let entry: T | undefined;
-                    if (!line) return;
-                    try {
-                      entry = JSON.parse(line);
-                    } catch (err) {
-                      console.log(line);
-                      await plugins.smartdelay.delayFor(10000);
-                    }
-                    if (!entry) return;
-                    totalRecordsCounter++;
-                    if (totalRecordsCounter % 10000 === 0)
-                      console.log(`${totalRecordsCounter} total records.`);
-                    await this.forEachFunction(entry);
-                  },
-                  1000
-                );
-                await concurrentProcessor.process(lines);
-              },
-              finalFunction: async (streamToolsArg) => {
-                console.log(`finished processing ${totalRecordsCounter} records.`);
-                if (nextRest) {
-                  JSON.parse(nextRest);
-                };
-                done.resolve();
-              },
-            })
-          );
-        },
-      })
-    );
-    await done.promise;
-  }
-}

package/ts/classes.main.opendata.ts

@@ -1,120 +0,0 @@
-import { BusinessRecord } from './classes.businessrecord.js';
-import { HandelsRegister } from './classes.handelsregister.js';
-import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
-import * as plugins from './plugins.js';
-
-export interface IOpenDataConfig {
-  downloadDir: string;
-  germanBusinessDataDir: string;
-  nogitDir: string;
-}
-
-export class OpenData {
-  public db!: plugins.smartdata.SmartdataDb;
-  private localSmartDb?: plugins.smartdb.LocalSmartDb;
-  private config: IOpenDataConfig;
-  private started = false;
-
-  public jsonLDataProcessor!: JsonlDataProcessor<SeedEntryType>;
-  public handelsregister!: HandelsRegister;
-
-  public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
-
-  constructor(configArg: IOpenDataConfig) {
-    if (!configArg) {
-      throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.');
-    }
-    if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) {
-      throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
-    }
-    this.config = configArg;
-  }
-
-  public async start() {
-    if (this.started) {
-      return;
-    }
-
-    // Ensure configured directories exist
-    await plugins.smartfs.directory(this.config.nogitDir).create();
-    await plugins.smartfs.directory(this.config.downloadDir).create();
-    await plugins.smartfs.directory(this.config.germanBusinessDataDir).create();
-
-    this.localSmartDb = new plugins.smartdb.LocalSmartDb({
-      folderPath: plugins.path.join(this.config.nogitDir, 'opendata-smartdb'),
-    });
-
-    const connectionInfo = await this.localSmartDb.start();
-    this.db = new plugins.smartdata.SmartdataDb({
-      mongoDbUrl: connectionInfo.connectionUri,
-      mongoDbName: 'opendata',
-    });
-
-    try {
-      await this.db.init();
-      await this.db.mongoDb.collection('_opendata_bootstrap').insertOne({
-        createdAt: new Date(),
-      });
-      await this.db.mongoDb.collection('_opendata_bootstrap').deleteMany({});
-
-      this.jsonLDataProcessor = new JsonlDataProcessor(
-        this.config.germanBusinessDataDir,
-        async (entryArg) => {
-          const businessRecord = new this.CBusinessRecord();
-          businessRecord.id = await this.CBusinessRecord.getNewId();
-          businessRecord.data.name = entryArg.name;
-          businessRecord.data.germanParsedRegistration = {
-            court: entryArg.all_attributes.registered_office,
-            number: entryArg.all_attributes._registerNummer,
-            type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
-          };
-          await businessRecord.save();
-        }
-      );
-
-      this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
-      await this.handelsregister.start();
-      this.started = true;
-    } catch (error) {
-      if (this.handelsregister) {
-        await this.handelsregister.stop().catch(() => {});
-      }
-      await this.db.close().catch(() => {});
-      await this.localSmartDb.stop().catch(() => {});
-      this.localSmartDb = undefined;
-      throw error;
-    }
-  }
-
-  public async buildInitialDb() {
-    await this.jsonLDataProcessor.processDataFromUrl();
-  }
-
-  public async slowValidateDb() {
-
-  }
-
-  public async validateSearchByName() {
-
-  }
-
-  public async searchDbByBusinessNameAndPostalCode(businessNameArg: string, postalCodeArg: string) {
-
-  }
-
-
-
-  public async stop() {
-    if (!this.started) {
-      return;
-    }
-
-    if (this.handelsregister) {
-      await this.handelsregister.stop();
-    }
-    await this.db.close();
-    await this.localSmartDb?.stop();
-    this.localSmartDb = undefined;
-    this.started = false;
-  }
-}

package/ts/laws/classes.lawrecord.ts

@@ -1,81 +0,0 @@
-import * as plugins from '../plugins.js';
-import type { TJurisdiction, TLawSource, TRawLawFormat } from './interfaces.law.js';
-
-@plugins.smartdata.Manager()
-export class LawRecord extends plugins.smartdata.SmartDataDbDoc<LawRecord, LawRecord> {
-  public static getByLookupKey = async (lookupKeyArg: string) => {
-    const lawRecords = await LawRecord.getInstances({
-      lookupKey: lookupKeyArg,
-    });
-    return lawRecords[0];
-  };
-
-  @plugins.smartdata.unI()
-  id!: string;
-
-  @plugins.smartdata.unI()
-  @plugins.smartdata.svDb()
-  lookupKey!: string;
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  jurisdiction!: TJurisdiction;
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  source!: TLawSource;
-
-  @plugins.smartdata.searchable()
-  @plugins.smartdata.svDb()
-  identifier!: string;
-
-  @plugins.smartdata.searchable()
-  @plugins.smartdata.svDb()
-  title!: string;
-
-  @plugins.smartdata.searchable()
-  @plugins.smartdata.svDb()
-  shortTitle: string = '';
-
-  @plugins.smartdata.searchable()
-  @plugins.smartdata.svDb()
-  citation: string = '';
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  type: string = '';
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  language: string = '';
-
-  @plugins.smartdata.svDb()
-  sourceUrl!: string;
-
-  @plugins.smartdata.svDb()
-  rawFormat!: TRawLawFormat;
-
-  @plugins.smartdata.svDb()
-  rawBody!: string;
-
-  @plugins.smartdata.searchable()
-  @plugins.smartdata.svDb()
-  text!: string;
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  dateIssued: string = '';
-
-  @plugins.smartdata.index()
-  @plugins.smartdata.svDb()
-  lastModified: string = '';
-
-  @plugins.smartdata.svDb()
-  sourceMeta: Record<string, string> = {};
-
-  @plugins.smartdata.svDb()
-  fetchedAt: Date = new Date();
-
-  @plugins.smartdata.svDb()
-  syncedAt: Date = new Date();
-}