@fin.cx/opendata 1.4.4 → 1.5.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@
3
3
  */
4
4
  export const commitinfo = {
5
5
  name: '@fin.cx/opendata',
6
- version: '1.4.4',
7
- description: 'A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.'
6
+ version: '1.5.3',
7
+ description: 'A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.'
8
8
  };
9
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSxrQkFBa0I7SUFDeEIsT0FBTyxFQUFFLE9BQU87SUFDaEIsV0FBVyxFQUFFLDBJQUEwSTtDQUN4SixDQUFBIn0=
9
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSxrQkFBa0I7SUFDeEIsT0FBTyxFQUFFLE9BQU87SUFDaEIsV0FBVyxFQUFFLHFOQUFxTjtDQUNuTyxDQUFBIn0=
@@ -50,9 +50,10 @@ export class JsonlDataProcessor {
50
50
  },
51
51
  finalFunction: async (streamToolsArg) => {
52
52
  console.log(`finished processing ${totalRecordsCounter} records.`);
53
- if (!nextRest)
54
- return;
55
- JSON.parse(nextRest);
53
+ if (nextRest) {
54
+ JSON.parse(nextRest);
55
+ }
56
+ ;
56
57
  done.resolve();
57
58
  },
58
59
  }));
@@ -61,4 +62,4 @@ export class JsonlDataProcessor {
61
62
  await done.promise;
62
63
  }
63
64
  }
64
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5qc29ubGRhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9jbGFzc2VzLmpzb25sZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGNBQWMsQ0FBQztBQUN4QyxPQUFPLEtBQUssS0FBSyxNQUFNLFlBQVksQ0FBQztBQXlDcEMsTUFBTSxPQUFPLGtCQUFrQjtJQUU3QixZQUFZLGtCQUErQztRQUN6RCxJQUFJLENBQUMsZUFBZSxHQUFHLGtCQUFrQixDQUFDO0lBQzVDLENBQUM7SUFFRCw4REFBOEQ7SUFDdkQsS0FBSyxDQUFDLGtCQUFrQixDQUM3QixVQUFVLEdBQUcsK0RBQStEO1FBRTVFLE1BQU0sSUFBSSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDMUMsTUFBTSxVQUFVLEdBQUcsTUFBTSxPQUFPLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxXQUFXLENBQUMsS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUM7UUFDdkYsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1lBQ2hCLE1BQU0sT0FBTyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDO1FBQ3BFLENBQUM7YUFBTSxDQUFDO1FBQ1IsQ0FBQztRQUVELE1BQU0sWUFBWSxHQUFHLE1BQU0sT0FBTyxDQUFDLFlBQVksQ0FBQyxZQUFZLENBQUMsY0FBYyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3hGLE1BQU0sZUFBZSxHQUFHLE1BQU0sWUFBWSxDQUFDLDJCQUEyQixFQUFFLENBQUM7UUFDekUsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsSUFBSSxRQUFRLEdBQVcsRUFBRSxDQUFDO1FBQzFCLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7WUFDbEMsVUFBVSxFQUFFLElBQUk7WUFDaEIsYUFBYSxFQUFFLEtBQUssRUFBRSxRQUFzQyxFQUFFLGNBQWMsRUFBRSxFQUFFO2dCQUM5RSxNQUFNLFVBQVUsR0FBRyxNQUFNLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO2dCQUNyRCxVQUFVLENBQUMsSUFBSSxDQUNiLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7b0JBQ2xDLFVBQVUsRUFBRSxJQUFJO29CQUNoQixhQUFhLEVBQUUsS0FBSyxFQUFFLFFBQWdCLEVBQUUsY0FBYyxFQUFFLEVBQUU7d0JBQ3hELE1BQU0sYUFBYSxHQUFHLFFBQVEsR0FBRyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7d0JBQ3JELE1BQU0sS0FBSyxHQUFHLGFBQWEsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3hDLFFBQVEsR0FBRyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUM7d0JBQ3ZCLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxLQUFLLENBQUMsTUFBTSxXQUFXLENBQUMsQ0FBQzt3QkFDcEQsTUFBTSxtQkFBbUIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsbUJBQW1CLENBQ3BFLEtBQUssRUFBRSxJQUFJLEVBQUUsRUFBRTs0QkFDYixJQUFJLEtBQVEsQ0FBQzs0QkFDYixJQUFJLENBQUMsSUFBSTtnQ0FBRSxPQUFPOzRCQUNsQixJQUFJLENBQUM7Z0NBQ0gsS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7NEJBQzNCLENBQUM7NEJBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQ0FDYixPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO2dDQUNsQixNQUFNLE9BQU8sQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDOzRCQUMzQyxDQUFDOzRCQUNELElBQUksQ0FBQyxLQUFLO2dDQUFFLE9BQU87NEJBQ25CLG1CQUFtQixFQUFFLENBQUM7NEJBQ3RCLElBQUksbUJBQW1CLEdBQUcsS0FBSyxLQUFLLENBQUM7Z0NBQ25DLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxtQkFBbUIsaUJBQWlCLENBQUMsQ0FBQzs0QkFDdkQsTUFBTSxJQUFJLENBQUMsZUFBZSxDQUFDLEtBQUssQ0FBQyxDQUFDO3dCQUNwQyxDQUFDLEVBQ0QsSUFBSSxDQUNMLENBQUM7d0JBQ0YsTUFBTSxtQkFBbUIsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQzNDLENBQUM7b0JBQ0QsYUFBYSxFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsRUFBRTt3QkFDdEMsT0FBTyxDQUFDLEdBQUcsQ0FBQyx1QkFBdUIsbUJBQW1CLFdBQVcsQ0FBQyxDQUFDO3dCQUNuRSxJQUFJLENBQUMsUUFBUTs0QkFBRSxPQUFPO3dCQUN0QixJQUFJLENBQUMsS0FBSyxDQUFDLFFBQVEsQ0FBQyxDQUFDO3dCQUNyQixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2pCLENBQUM7aUJBQ0YsQ0FBQyxDQUNILENBQUM7WUFDSixDQUFDO1NBQ0YsQ0FBQyxDQUNILENBQUM7UUFDRixNQUFNLElBQUksQ0FBQyxPQUFPLENBQUM7SUFDckIsQ0FBQztDQUNGIn0=
65
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2xhc3Nlcy5qc29ubGRhdGEuanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi90cy9jbGFzc2VzLmpzb25sZGF0YS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssT0FBTyxNQUFNLGNBQWMsQ0FBQztBQUN4QyxPQUFPLEtBQUssS0FBSyxNQUFNLFlBQVksQ0FBQztBQXlDcEMsTUFBTSxPQUFPLGtCQUFrQjtJQUU3QixZQUFZLGtCQUErQztRQUN6RCxJQUFJLENBQUMsZUFBZSxHQUFHLGtCQUFrQixDQUFDO0lBQzVDLENBQUM7SUFFRCw4REFBOEQ7SUFDdkQsS0FBSyxDQUFDLGtCQUFrQixDQUM3QixVQUFVLEdBQUcsK0RBQStEO1FBRTVFLE1BQU0sSUFBSSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsS0FBSyxFQUFFLENBQUM7UUFDMUMsTUFBTSxVQUFVLEdBQUcsTUFBTSxPQUFPLENBQUMsU0FBUyxDQUFDLEVBQUUsQ0FBQyxXQUFXLENBQUMsS0FBSyxDQUFDLHFCQUFxQixDQUFDLENBQUM7UUFDdkYsSUFBSSxDQUFDLFVBQVUsRUFBRSxDQUFDO1lBQ2hCLE1BQU0sT0FBTyxDQUFDLFNBQVMsQ0FBQyxFQUFFLENBQUMsU0FBUyxDQUFDLEtBQUssQ0FBQyxxQkFBcUIsQ0FBQyxDQUFDO1FBQ3BFLENBQUM7YUFBTSxDQUFDO1FBQ1IsQ0FBQztRQUVELE1BQU0sWUFBWSxHQUFHLE1BQU0sT0FBTyxDQUFDLFlBQVksQ0FBQyxZQUFZLENBQUMsY0FBYyxDQUFDLFVBQVUsQ0FBQyxDQUFDO1FBQ3hGLE1BQU0sZUFBZSxHQUFHLE1BQU0sWUFBWSxDQUFDLDJCQUEyQixFQUFFLENBQUM7UUFDekUsSUFBSSxtQkFBbUIsR0FBRyxDQUFDLENBQUM7UUFDNUIsSUFBSSxRQUFRLEdBQVcsRUFBRSxDQUFDO1FBQzFCLGVBQWUsQ0FBQyxJQUFJLENBQ2xCLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7WUFDbEMsVUFBVSxFQUFFLElBQUk7WUFDaEIsYUFBYSxFQUFFLEtBQUssRUFBRSxRQUFzQyxFQUFFLGNBQWMsRUFBRSxFQUFFO2dCQUM5RSxNQUFNLFVBQVUsR0FBRyxNQUFNLFFBQVEsQ0FBQyxnQkFBZ0IsRUFBRSxDQUFDO2dCQUNyRCxVQUFVLENBQUMsSUFBSSxDQUNiLElBQUksT0FBTyxDQUFDLFdBQVcsQ0FBQyxXQUFXLENBQUM7b0JBQ2xDLFVBQVUsRUFBRSxJQUFJO29CQUNoQixhQUFhLEVBQUUsS0FBSyxFQUFFLFFBQWdCLEVBQUUsY0FBYyxFQUFFLEVBQUU7d0JBQ3hELE1BQU0sYUFBYSxHQUFHLFFBQVEsR0FBRyxRQUFRLENBQUMsUUFBUSxFQUFFLENBQUM7d0JBQ3JELE1BQU0sS0FBSyxHQUFHLGFBQWEsQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7d0JBQ3hDLFFBQVEsR0FBRyxLQUFLLENBQUMsR0FBRyxFQUFFLENBQUM7d0JBQ3ZCLE9BQU8sQ0FBQyxHQUFHLENBQUMsZUFBZSxLQUFLLENBQUMsTUFBTSxXQUFXLENBQUMsQ0FBQzt3QkFDcEQsTUFBTSxtQkFBbUIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxVQUFVLENBQUMsbUJBQW1CLENBQ3BFLEtBQUssRUFBRSxJQUFJLEVBQUUsRUFBRTs0QkFDYixJQUFJLEtBQVEsQ0FBQzs0QkFDYixJQUFJLENBQUMsSUFBSTtnQ0FBRSxPQUFPOzRCQUNsQixJQUFJLENBQUM7Z0NBQ0gsS0FBSyxHQUFHLElBQUksQ0FBQyxLQUFLLENBQUMsSUFBSSxDQUFDLENBQUM7NEJBQzNCLENBQUM7NEJBQUMsT0FBTyxHQUFHLEVBQUUsQ0FBQztnQ0FDYixPQUFPLENBQUMsR0FBRyxDQUFDLElBQUksQ0FBQyxDQUFDO2dDQUNsQixNQUFNLE9BQU8sQ0FBQyxVQUFVLENBQUMsUUFBUSxDQUFDLEtBQUssQ0FBQyxDQUFDOzRCQUMzQyxDQUFDOzRCQUNELElBQUksQ0FBQyxLQUFLO2dDQUFFLE9BQU87NEJBQ25CLG1CQUFtQixFQUFFLENBQUM7NEJBQ3RCLElBQUksbUJBQW1CLEdBQUcsS0FBSyxLQUFLLENBQUM7Z0NBQ25DLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxtQkFBbUIsaUJBQWlCLENBQUMsQ0FBQzs0QkFDdkQsTUFBTSxJQUFJLENBQUMsZUFBZSxDQUFDLEtBQUssQ0FBQyxDQUFDO3dCQUNwQyxDQUFDLEVBQ0QsSUFBSSxDQUNMLENBQUM7d0JBQ0YsTUFBTSxtQkFBbUIsQ0FBQyxPQUFPLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBQzNDLENBQUM7b0JBQ0QsYUFBYSxFQUFFLEtBQUssRUFBRSxjQUFjLEVBQUUsRUFBRTt3QkFDdEMsT0FBTyxDQUFDLEdBQUcsQ0FBQyx1QkFBdUIsbUJBQW1CLFdBQVcsQ0FBQyxDQUFDO3dCQUNuRSxJQUFJLFFBQVEsRUFBRSxDQUFDOzRCQUNiLElBQUksQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUM7d0JBQ3ZCLENBQUM7d0JBQUEsQ0FBQzt3QkFDRixJQUFJLENBQUMsT0FBTyxFQUFFLENBQUM7b0JBQ2pCLENBQUM7aUJBQ0YsQ0FBQyxDQUNILENBQUM7WUFDSixDQUFDO1NBQ0YsQ0FBQyxDQUNILENBQUM7UUFDRixNQUFNLElBQUksQ0FBQyxPQUFPLENBQUM7SUFDckIsQ0FBQztDQUNGIn0=
package/npmextra.json CHANGED
@@ -5,25 +5,26 @@
5
5
  "githost": "gitlab.com",
6
6
  "gitscope": "fin.cx",
7
7
  "gitrepo": "opendata",
8
- "description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
8
+ "description": "A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.",
9
9
  "npmPackagename": "@fin.cx/opendata",
10
10
  "license": "MIT",
11
11
  "projectDomain": "fin.cx",
12
12
  "keywords": [
13
13
  "TypeScript",
14
14
  "open data",
15
- "business data",
16
15
  "German companies",
17
- "data management",
18
- "business registry",
19
- "npm package",
16
+ "business data",
20
17
  "MongoDB",
18
+ "JSONL",
19
+ "bulk processing",
20
+ "data management",
21
21
  "automation",
22
- "data integration",
23
- "database",
24
- "data processing",
25
- "data retrieval",
26
- "data update"
22
+ "browser automation",
23
+ "Handelsregister",
24
+ "web scraping",
25
+ "file processing",
26
+ "business registry",
27
+ "data retrieval"
27
28
  ]
28
29
  }
29
30
  },
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "@fin.cx/opendata",
3
- "version": "1.4.4",
3
+ "version": "1.5.3",
4
4
  "private": false,
5
- "description": "A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.",
5
+ "description": "A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.",
6
6
  "main": "dist_ts/index.js",
7
7
  "typings": "dist_ts/index.d.ts",
8
8
  "type": "module",
@@ -35,12 +35,12 @@
35
35
  },
36
36
  "repository": {
37
37
  "type": "git",
38
- "url": "git+https://gitlab.com/fin.cx/opendata.git"
38
+ "url": "https://code.foss.global/fin.cx/opendata.git"
39
39
  },
40
40
  "bugs": {
41
- "url": "https://gitlab.com/fin.cx/opendata/issues"
41
+ "url": "https://code.foss.global/fin.cx/opendata/issues"
42
42
  },
43
- "homepage": "https://gitlab.com/fin.cx/opendata#readme",
43
+ "homepage": "https://code.foss.global/fin.cx/opendata#readme",
44
44
  "browserslist": [
45
45
  "last 1 chrome versions"
46
46
  ],
@@ -59,18 +59,19 @@
59
59
  "keywords": [
60
60
  "TypeScript",
61
61
  "open data",
62
- "business data",
63
62
  "German companies",
64
- "data management",
65
- "business registry",
66
- "npm package",
63
+ "business data",
67
64
  "MongoDB",
65
+ "JSONL",
66
+ "bulk processing",
67
+ "data management",
68
68
  "automation",
69
- "data integration",
70
- "database",
71
- "data processing",
72
- "data retrieval",
73
- "data update"
69
+ "browser automation",
70
+ "Handelsregister",
71
+ "web scraping",
72
+ "file processing",
73
+ "business registry",
74
+ "data retrieval"
74
75
  ],
75
76
  "scripts": {
76
77
  "test": "(tstest test/ --web)",
package/readme.md CHANGED
@@ -1,220 +1,649 @@
1
1
  # @fin.cx/opendata
2
-
3
- A TypeScript-based library for accessing and managing open business data, specifically for German companies.
2
+ A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.
4
3
 
5
4
  ## Install
6
5
 
7
- To install the `@fin.cx/opendata` package, you can use npm or yarn as your package manager. Here's how you can do it:
6
+ To install the @fin.cx/opendata package, you can use npm or yarn as your package manager. The installation process is simple and straightforward.
8
7
 
9
8
  Using npm:
10
9
 
11
- ```bash
12
- npm install @fin.cx/opendata
13
- ```
10
+ npm install @fin.cx/opendata
14
11
 
15
12
  Using yarn:
16
13
 
17
- ```bash
18
- yarn add @fin.cx/opendata
19
- ```
14
+ yarn add @fin.cx/opendata
20
15
 
21
16
  ## Usage
22
17
 
23
- The `@fin.cx/opendata` package provides a comprehensive set of functionalities for handling open business data, focusing on German business data. Let's explore the detailed capabilities of this library through extensive examples and instructions.
18
+ The @fin.cx/opendata library is a versatile tool that empowers developers to integrate comprehensive open business data into their systems. This library is particularly tailored for German companies, offering functionalities that include creating, retrieving, updating, and deleting business records as well as processing large volumes of JSONL data from external sources. In addition to core database operations via MongoDB, the library provides integration with web-based services, primarily through a hands-on Handelsregister processor that utilizes browser automation for searching and downloading documents.
19
+
20
+ In this section, we will extensively detail multiple usage scenarios, ensuring that every feature the module offers is thoroughly explored. All examples in this documentation employ ECMAScript Module (ESM) syntax and TypeScript, highlighting proper asynchronous handling, error management, and advanced integration with other dependencies. We will walk you through environment setup, initializing the package, managing business records, processing bulk JSONL data, interacting with the Handelsregister for on-demand document retrieval, and much more. Each example is constructed to expose every nuance of the module's behavior and usage.
21
+
22
+ For clarity, we will split this section into multiple parts:
24
23
 
25
- ### Setting Up the Environment
24
+ 1. Environment Setup and Initializing the Library
25
+ 2. Managing Business Records (CRUD Operations)
26
+ 3. Bulk Data Processing and Importing via JSONL Streams
27
+ 4. Integrating with the Handelsregister: Detailed Demonstrations
28
+ 5. Advanced Examples: Combined Operations and Edge Cases
29
+ 6. Error Handling and Data Validation
30
+ 7. Testing and Automated Workflows
26
31
 
27
- First, make sure you've set up the necessary environment variables for MongoDB. You will need the following environment variables:
28
- - `MONGODB_URL`: The URL for your MongoDB instance.
29
- - `MONGODB_NAME`: The name of the database to use.
30
- - `MONGODB_USER`: A valid username for accessing the database.
31
- - `MONGODB_PASS`: The password associated with the MongoDB user.
32
+ Throughout these examples, we will examine how each class and method interacts with the underlying MongoDB database and the system's file structure. We assume you have a running MongoDB instance and that your environment is configured with the necessary variables.
32
33
 
33
- These variables can be configured in a `.env` file or managed through a specific service used for secure environment variables handling.
34
+ ────────────────────────────────────────────
35
+ ### 1. Environment Setup and Initializing the Library
34
36
 
35
- ### Importing and Initializing the Library
37
+ Before diving into any operations, ensure that your development environment is properly configured. The @fin.cx/opendata library mandates several environment variables for connecting to your MongoDB instance. For a smooth experience, it is advisable to use a .env file or any secure secrets management tool that suits your workflow. The required environment variables include:
38
+
39
+ • MONGODB_URL – The connection string URL for your MongoDB instance.
40
+ • MONGODB_NAME – The name of the database that the module will interact with.
41
+ • MONGODB_USER – MongoDB username required for authentication.
42
+ • MONGODB_PASS – The password for the MongoDB user.
43
+
44
+ Below is an example .env file for local development:
45
+
46
+ ```
47
+ MONGODB_URL=mongodb://localhost:27017
48
+ MONGODB_NAME=opendataDB
49
+ MONGODB_USER=myUser
50
+ MONGODB_PASS=myPass
51
+ ```
36
52
 
37
- To start working with the library, import the necessary classes and initialize the `OpenData` class.
53
+ Once these variables are set, the library can fetch them using the integrated qenv tool. The following code snippet demonstrates how to import and initialize the library:
38
54
 
39
55
  ```typescript
40
56
  import { OpenData } from '@fin.cx/opendata';
41
57
 
42
- const initializeOpenData = async () => {
58
+ const startOpenDataInstance = async () => {
43
59
  const openData = new OpenData();
44
60
 
45
61
  try {
62
+ console.log('Starting OpenData instance...');
46
63
  await openData.start();
47
- console.log('OpenData instance has started successfully.');
48
-
49
- // Example usage:
50
- await createAndManageBusinessRecords(openData);
64
+ console.log('OpenData instance started successfully.');
65
+
66
+ // Invoke sample operations:
67
+ await demonstrateBusinessRecordsOperations(openData);
68
+ await demonstrateBulkDataProcessing(openData);
69
+ await demonstrateHandelsregisterOperations(openData);
51
70
 
52
71
  } catch (error) {
53
- console.error('Error starting OpenData:', error);
72
+ console.error('Error during initialization:', error);
54
73
  } finally {
74
+ console.log('Stopping OpenData instance...');
55
75
  await openData.stop();
56
- console.log('OpenData instance has stopped.');
76
+ console.log('OpenData instance stopped.');
57
77
  }
58
78
  };
59
79
 
60
- initializeOpenData();
80
+ startOpenDataInstance();
61
81
  ```
62
82
 
63
- ### Managing Business Records
83
+ In this snippet, we import the OpenData class from the module and execute its start and stop routines to ensure that the MongoDB connection is properly initialized and terminated. Notice that we move on to different demonstration functions that showcase individual features.
84
+
85
+ ────────────────────────────────────────────
86
+ ### 2. Managing Business Records (CRUD Operations)
64
87
 
65
- The `BusinessRecord` class represents a company's data. Let's explore how you can create, retrieve, update, and manage these records.
88
+ Central to the @fin.cx/opendata library is the management of business records. The BusinessRecord class encapsulates data pertaining to companies, allowing you to create new records, retrieve existing ones, update information, and delete entries when necessary. The following examples illustrate each operation within a robust context.
66
89
 
67
- #### Creating a New BusinessRecord
90
+ #### a) Creating a Business Record
68
91
 
69
- Creating a new business record involves instantiating the `BusinessRecord` class and setting the relevant properties.
92
+ Creating a new business record in the openData instance is straightforward. You instantiate a new record and populate its data properties with relevant details such as company name, address, registration number, managing directors, and much more. The sample below uses the embedded CBusinessRecord manager to generate a new record:
70
93
 
71
94
  ```typescript
72
- import { BusinessRecord } from '@fin.cx/opendata';
95
+ import { OpenData } from '@fin.cx/opendata';
73
96
 
74
- const createBusinessRecord = async (openData: OpenData) => {
97
+ export const createBusinessRecordExample = async (openData: OpenData) => {
75
98
  const businessRecord = new openData.CBusinessRecord();
76
-
99
+
77
100
  businessRecord.data = {
78
- name: "Tech Innovations GmbH",
79
- address: "Tech Park 42",
80
- postalCode: "80333",
81
- city: "Munich",
101
+ name: "Innovative Solutions GmbH",
102
+ address: "Musterstraße 1",
103
+ postalCode: "10115",
104
+ city: "Berlin",
82
105
  country: "Germany",
83
- phone: "+49 89 123456",
84
- email: "info@techinnovations.de",
85
- website: "https://techinnovations.de",
106
+ phone: "+49 30 123456",
107
+ email: "contact@innovativesolutions.de",
108
+ website: "https://innovativesolutions.de",
86
109
  businessType: "GmbH",
87
- registrationNumber: "HRB 654321",
88
- registrationCourt: "Munich",
110
+ registrationId: "District court Berlin HRB 987654",
89
111
  legalForm: "GmbH",
90
- managingDirectors: ["Alice Müller", "Bob Schmidt"],
91
- foundingDate: new Date("2020-01-01").toISOString(),
92
- capital: "100,000 EUR",
93
- purpose: "Developing innovative tech solutions",
112
+ managingDirectors: ["Max Mustermann", "Erika Musterfrau"],
113
+ foundingDate: new Date("2018-05-10").toISOString(),
114
+ capital: "250,000 EUR",
115
+ purpose: "Technology development and consulting services",
94
116
  lastUpdate: new Date().toISOString()
95
117
  };
96
118
 
97
- await businessRecord.save();
98
- console.log('BusinessRecord saved:', businessRecord);
119
+ try {
120
+ await businessRecord.save();
121
+ console.log('BusinessRecord created successfully:', businessRecord);
122
+ } catch (error) {
123
+ console.error('Error creating business record:', error);
124
+ }
99
125
  };
100
126
  ```
101
127
 
102
- #### Retrieving Business Records
128
+ In this example, after setting the business record fields, the record is saved to the MongoDB collection using the save method. The system ensures that the newly created record receives a unique identifier by generating a new ID when saving the document.
103
129
 
104
- You can retrieve existing business records by querying the database using various data fields.
130
+ #### b) Retrieving Business Records
131
+
132
+ To retrieve business records, you can search by various fields such as city, business name, or registration details. The system utilizes MongoDB queries to filter and return relevant documents. Below is a sample function that retrieves all records for companies based in a particular city:
105
133
 
106
134
  ```typescript
107
- const retrieveBusinessRecords = async (openData: OpenData) => {
108
- const records = await openData.db
109
- .collection<BusinessRecord>('businessrecords')
110
- .find({ city: "Munich" })
111
- .toArray();
135
+ import { OpenData } from '@fin.cx/opendata';
136
+ import type { BusinessRecord } from '@fin.cx/opendata';
112
137
 
113
- console.log('Found Business Records:', records);
138
+ export const retrieveRecordsByCity = async (openData: OpenData, city: string) => {
139
+ try {
140
+ const records = await openData.db
141
+ .collection<BusinessRecord>('businessrecords')
142
+ .find({ city })
143
+ .toArray();
144
+
145
+ console.log(`Retrieved ${records.length} records for city ${city}.`);
146
+ console.log(records);
147
+ return records;
148
+ } catch (error) {
149
+ console.error('Error retrieving business records:', error);
150
+ }
114
151
  };
115
152
  ```
116
153
 
117
- #### Updating Existing Records
154
+ This method queries the "businessrecords" collection using a simple filter and converts the cursor into an array of records. You can extend the query to filter by more sophisticated criteria as needed.
155
+
156
+ #### c) Updating an Existing Business Record
118
157
 
119
- To update an existing `BusinessRecord`, you retrieve the record, modify its data, and save it again.
158
+ Modifying the details of an exisiting record is a common operation. First, you need to retrieve the record from the database. Once the record is loaded, you can make changes to its properties and then save the updated record back to the database. The following example demonstrates this with a change to the company’s phone number and last update timestamp:
120
159
 
121
160
  ```typescript
122
- const updateBusinessRecord = async (openData: OpenData, recordId: string) => {
123
- const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
124
- if (businessRecord) {
125
- businessRecord.data.phone = "+49 89 987654";
126
- businessRecord.data.lastUpdate = new Date().toISOString();
161
+ import { OpenData } from '@fin.cx/opendata';
127
162
 
163
+ export const updateBusinessRecordExample = async (openData: OpenData, recordId: string) => {
164
+ try {
165
+ // Retrieve the record by its id using the manager’s helper
166
+ const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
167
+ if (!businessRecord) {
168
+ console.log(`No business record found with id: ${recordId}`);
169
+ return;
170
+ }
171
+
172
+ // Update some fields
173
+ businessRecord.data.phone = "+49 30 654321";
174
+ businessRecord.data.lastUpdate = new Date().toISOString();
175
+
176
+ // Save the updated record into the database
128
177
  await businessRecord.save();
129
- console.log('BusinessRecord updated:', businessRecord);
130
- } else {
131
- console.log('BusinessRecord not found for id:', recordId);
178
+ console.log("Business record updated successfully:", businessRecord);
179
+ } catch (error) {
180
+ console.error('Error updating business record:', error);
132
181
  }
133
182
  };
134
183
  ```
135
184
 
136
- #### Deleting a Business Record
185
+ This code snippet presents a robust pattern where errors are caught and logged, ensuring that any update issues can be diagnosed easily.
137
186
 
138
- You can delete a business record using its unique identifier.
187
+ #### d) Deleting a Business Record
188
+
189
+ The deletion of a record is as vital as its creation and modification. The library provides a delete method that removes the specified record from the database. Below is a simple function to delete a business record by its identifier:
139
190
 
140
191
  ```typescript
141
- const deleteBusinessRecord = async (openData: OpenData, recordId: string) => {
142
- const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
143
- if (businessRecord) {
144
- await businessRecord.delete();
145
- console.log(`BusinessRecord with id ${recordId} deleted successfully.`);
146
- } else {
147
- console.log('No record found for the provided id:', recordId);
192
+ import { OpenData } from '@fin.cx/opendata';
193
+
194
+ export const deleteBusinessRecordExample = async (openData: OpenData, recordId: string) => {
195
+ try {
196
+ const businessRecord = await openData.CBusinessRecord.getInstance(recordId);
197
+ if (businessRecord) {
198
+ await businessRecord.delete();
199
+ console.log(`Successfully deleted business record with id: ${recordId}`);
200
+ } else {
201
+ console.log(`No business record found with id: ${recordId}`);
202
+ }
203
+ } catch (error) {
204
+ console.error('Error deleting business record:', error);
148
205
  }
149
206
  };
150
207
  ```
151
208
 
152
- ### Updating German Business Data
209
+ Through this example, you can integrate safe deletion practices in your application, removing outdated or incorrect records without compromising database integrity.
210
+
211
+ ────────────────────────────────────────────
212
+ ### 3. Bulk Data Processing and Importing via JSONL Streams
153
213
 
154
- The package includes functionalities to keep your business data up-to-date by downloading from official German open data repositories.
214
+ One of the powerful features of the @fin.cx/opendata module is its ability to process large datasets provided in the JSON Lines (JSONL) format. The JsonlDataProcessor class is designed to handle streaming data, processing each record concurrently, and efficiently updating the database.
215
+
216
+ This bulk data ingestion mechanism is particularly useful when dealing with large-scale datasets such as the German companies' open data that the module fetches from official data portals. The process involves decompressing, streaming, and parsing data by leveraging pipelines of smart streams and concurrent processors.
217
+
218
+ Below is an extended example demonstrating how to process a JSONL data file from a given URL:
155
219
 
156
220
  ```typescript
157
- const updateGermanBusinessData = async (openData: OpenData) => {
221
+ import { OpenData } from '@fin.cx/opendata';
222
+
223
+ // This function demonstrates bulk data processing using the JSONL data processor.
224
+ // The dataUrl parameter is optional and, if not provided, defaults to the official open data URL.
225
+ export const demonstrateBulkDataProcessing = async (openData: OpenData, dataUrl?: string) => {
226
+ try {
227
+ console.log('Starting bulk data processing...');
228
+ await openData.jsonLDataProcessor.processDataFromUrl(dataUrl);
229
+ console.log('Bulk data processing completed successfully.');
230
+ } catch (error) {
231
+ console.error('Error during bulk data processing:', error);
232
+ }
233
+ };
234
+ ```
235
+
236
+ In the processDataFromUrl implementation, the library uses a pipeline of smart streams. After downloading the compressed file, it decompresses it and splits the content into discrete JSON lines. The processor then concurrently applies a handler function to each JSON entry. This function extracts relevant company details, instantiates a new BusinessRecord, associates parsed data (for example, registration attributes from German registers), and saves the record to MongoDB.
237
+
238
+ A deeper dive into the processing mechanism:
239
+ • The JSONL data is received as a binary (Buffer) stream.
240
+ • The stream is piped into a duplex stream that splits the text by newline characters.
241
+ • Each line is parsed into a JSON object and passed into an asynchronous processing function.
242
+ • This function creates a new business record and sets properties such as the company name and its registration details, derived from the JSON entry.
243
+ • As the processor moves through the stream, it logs progress every 10,000 records to give feedback on its bulk processing status.
244
+
245
+ By supporting concurrency (with a configurable concurrency limit, e.g., 1000 simultaneous operations), the library ensures that even gigabytes of data are processed efficiently without hitting memory bottlenecks.
246
+
247
+ ────────────────────────────────────────────
248
+ ### 4. Integrating with the Handelsregister: Detailed Demonstrations
249
+
250
+ In addition to CRUD operations and bulk processing, the module includes an integrated Handelsregister system. This sophisticated component leverages a headless browser (via the smartbrowser instance) to interact with the official Handelsregister website. Through this integration, you can search for companies, navigate to specific pages, trigger file downloads (such as PDF or XML data), and parse the downloaded content for further processing.
251
+
252
+ #### a) Starting the Handelsregister
253
+
254
+ Before executing any search or download operations, the Handelsregister system must be started. The start method initializes required resources including starting a headless browser, ensuring download directories are created, and preparing asynchronous stacks for exclusive execution.
255
+
256
+ ```typescript
257
+ import { OpenData } from '@fin.cx/opendata';
258
+
259
+ export const demonstrateHandelsregisterStart = async (openData: OpenData) => {
158
260
  try {
159
- await openData.germanBusinesses.update();
160
- console.log('German business data has been updated successfully.');
261
+ console.log('Starting Handelsregister services...');
262
+ await openData.handelsregister.start();
263
+ console.log('Handelsregister ready.');
161
264
  } catch (error) {
162
- console.error('Error updating German business data:', error);
265
+ console.error('Error starting Handelsregister service:', error);
163
266
  }
164
267
  };
165
268
  ```
166
269
 
167
- This function fetches the latest open data regarding German companies, processes it, and updates your local database.
270
+ #### b) Searching for a Company Using the Handelsregister
271
+
272
+ A common use case is to search for a company by its name. The Handelsregister system creates a dedicated browser page, enters the search criteria into the input fields, selects the appropriate options (such as radio buttons for search type), and clicks the “Find” button. The following function demonstrates how to incorporate these actions:
273
+
274
+ ```typescript
275
+ import { OpenData } from '@fin.cx/opendata';
168
276
 
169
- ### Detailed Class Insights
277
+ export const searchCompanyExample = async (openData: OpenData, companyName: string, limit: number = 100) => {
278
+ try {
279
+ console.log(`Searching for company with name "${companyName}"...`);
280
+ const records = await openData.handelsregister.searchCompany(companyName, limit);
281
+ console.log(`Found ${records.length} matching records for "${companyName}".`);
282
+ console.log('Records:', records);
283
+ return records;
284
+ } catch (error) {
285
+ console.error('Error searching for company:', error);
286
+ }
287
+ };
288
+ ```
170
289
 
171
- #### OpenData Class
290
+ In this scenario, the Handelsregister component uses internal helper functions:
291
+ • getNewPage – to create a new browser page with file download behavior enabled.
292
+ • navigateToPage – which navigates to the “Normal search” page.
293
+ • Input events – to simulate typing in search criteria.
294
+ • UI interactions – to select options and trigger the search.
172
295
 
173
- The `OpenData` class serves as the core of the library, initializing necessary components and controlling data flows:
296
+ #### c) Retrieving Detailed Data and Triggering Downloads
174
297
 
175
- - **db**: Represents the connection to your MongoDB database.
176
- - **germanBusinesses**: An instance handling specific operations related to German business data updates.
298
+ After obtaining general search results, you may wish to retrieve more detailed information about a specific company. Provided you have the parsed registration data (which typically includes the registration court, type, and number), you can instruct the system to navigate to a detailed view and trigger file downloads. These files might include the company’s official registry entry (as an XML file) and additional documents (such as a PDF summary).
299
+
300
+ The example below details how to use the Handelsregister functionality to focus on a specific company by leveraging its registration details, then download both SI and AD files:
177
301
 
178
302
  ```typescript
179
- class OpenData {
180
- db: plugins.smartdata.SmartdataDb;
181
- germanBusinesses: GermanBusinessData;
303
+ import { OpenData } from '@fin.cx/opendata';
304
+
305
+ export const getDetailedCompanyData = async (openData: OpenData, registrationData: { court?: string; type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR'; number?: string; }) => {
306
+ try {
307
+ console.log('Retrieving detailed company data...');
308
+ const result = await openData.handelsregister.getSpecificCompany(registrationData);
309
+ console.log('Retrieved detailed company data.');
310
+ console.log('Business Records:', result.businessRecords);
311
+ console.log('Downloaded Files:', result.files);
312
+ return result;
313
+ } catch (error) {
314
+ console.error('Error retrieving detailed company data:', error);
315
+ }
316
+ };
317
+ ```
318
+
319
+ In the above example:
320
+ • The getSpecificCompany method triggers navigation through various UI elements:
321
+ – Selecting the register type via a dropdown.
322
+ – Inputting the register number.
323
+ – Choosing the appropriate register court.
324
+ • Then, after clicking the “Find” button, the system waits for the results, verifies the visual components on the page, and initiates file downloads.
325
+ • The downloaded files are renamed according to their type (SI for XML and AD for PDF) and are returned for further processing or storage.
182
326
 
183
- private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
327
+ #### d) Downloading and Processing Files
184
328
 
185
- public async start() {
186
- // Database initialization logic
329
+ The Handelsregister component not only triggers file downloads but also includes utility functions that wait for downloads to complete, clear temporary directories, and output the file objects. You may want to use these file objects to persist data locally, parse file content, or send the data downstream for further analysis.
330
+
331
+ Below is an example that covers downloading and saving the files into a custom directory for post-download analysis:
332
+
333
+ ```typescript
334
+ import { OpenData } from '@fin.cx/opendata';
335
+ import * as path from 'path';
336
+
337
+ export const downloadAndSaveFilesExample = async (openData: OpenData, registrationData: { court?: string; type?: 'HRA' | 'HRB' | 'GnR' | 'PR' | 'VR' | 'GsR'; number?: string; }) => {
338
+ try {
339
+ console.log('Initiating specific company download...');
340
+ const result = await openData.handelsregister.getSpecificCompany(registrationData);
341
+ const saveDirectory = path.join(process.cwd(), 'downloaded_files');
342
+
343
+ // Save each downloaded file to the specified directory
344
+ for (const file of result.files) {
345
+ await file.writeToDir(saveDirectory);
346
+ console.log(`File saved: ${file.path}`);
347
+ }
348
+ } catch (error) {
349
+ console.error('Error during file download and save process:', error);
187
350
  }
351
+ };
352
+ ```
353
+
354
+ This function demonstrates a complete flow from launching the Handelsregister detailed company search to saving the downloaded files to disk. This example is particularly useful in scenarios where the downloaded documents need to be processed further, such as converting XML to JSON or extracting text from PDFs.
188
355
 
189
- public async stop() {
190
- // Cleanup logic
356
+ ────────────────────────────────────────────
357
+ ### 5. Advanced Examples: Combined Operations and Edge Cases
358
+
359
+ Given the numerous functionalities offered by the library, you can combine various operations to create more complex workflows. One such example is an end-to-end pipeline that:
360
+ 1. Initializes the open data instance.
361
+ 2. Processes an initial bulk data import.
362
+ 3. Searches for key business records that match specific criteria.
363
+ 4. Updates individual records based on additional data retrieved from the Handelsregister.
364
+ 5. Handles error conditions and retries processes where necessary.
365
+
366
+ The following advanced example integrates these steps:
367
+
368
+ ```typescript
369
+ import { OpenData } from '@fin.cx/opendata';
370
+
371
+ const advancedWorkflowExample = async () => {
372
+ const openData = new OpenData();
373
+
374
+ try {
375
+ console.log('Starting advanced workflow...');
376
+ await openData.start();
377
+
378
+ // Step 1: Bulk data import from external JSONL source
379
+ console.log('Building initial database from bulk import...');
380
+ await openData.buildInitialDb();
381
+
382
+ // Step 2: Search for companies in a selected area (for instance, Munich)
383
+ console.log('Retrieving companies located in Munich...');
384
+ const munichRecords = await openData.db
385
+ .collection('businessrecords')
386
+ .find({ city: "Munich" })
387
+ .toArray();
388
+ console.log(`Found ${munichRecords.length} companies in Munich.`);
389
+
390
+ // Step 3: For each record, perform an update operation based on new file downloads
391
+ for (const record of munichRecords) {
392
+ try {
393
+ console.log(`Updating record for company: ${record.data.name}`);
394
+ // Assuming the record contains parsed registration info
395
+ if (record.data.germanParsedRegistration) {
396
+ const detailedData = await openData.handelsregister.getSpecificCompany(record.data.germanParsedRegistration);
397
+ // Update business record with new information (e.g., registration files or updated details)
398
+ record.data.lastUpdate = new Date().toISOString();
399
+ // You might want to add additional fields based on the downloaded file data
400
+ await record.save();
401
+ console.log(`Updated record for ${record.data.name}.`);
402
+ } else {
403
+ console.log(`No registration data available for ${record.data.name}; skipping update.`);
404
+ }
405
+ } catch (innerError) {
406
+ console.error(`Error updating record for ${record.data.name}:`, innerError);
407
+ }
408
+ }
409
+
410
+ // Step 4: Demonstrate retrieval and deletion
411
+ const recordToDeleteId = munichRecords.length > 0 ? munichRecords[0].id : null;
412
+ if (recordToDeleteId) {
413
+ console.log(`Deleting record with id: ${recordToDeleteId}`);
414
+ const recordToDelete = await openData.CBusinessRecord.getInstance(recordToDeleteId);
415
+ if (recordToDelete) {
416
+ await recordToDelete.delete();
417
+ console.log(`Record ${recordToDeleteId} deleted successfully.`);
418
+ }
419
+ }
420
+
421
+ } catch (error) {
422
+ console.error('Advanced workflow encountered an error:', error);
423
+ } finally {
424
+ console.log('Ending advanced workflow and stopping OpenData instance.');
425
+ await openData.stop();
191
426
  }
192
- }
427
+ };
428
+
429
+ advancedWorkflowExample();
193
430
  ```
194
431
 
195
- #### GermanBusinessData Class
432
+ This advanced workflow not only illustrates the coordinated use of bulk data import, search, update, and delete operations but also demonstrates the integration of browser automation for fetching detailed data. The error handling at each step ensures that even if a particular operation fails, the workflow continues in a controlled fashion.
433
+
434
+ ────────────────────────────────────────────
435
+ ### 6. Error Handling and Data Validation
196
436
 
197
- This class deals specifically with German company data — fetching, processing, and updating local databases consistently with official German data sources.
437
+ Robust systems must gracefully handle errors and ensure data consistency. The @fin.cx/opendata library has built-in error handling for asynchronous operations, whether connecting to MongoDB, processing JSON streams, or interacting with web pages. In addition, each BusinessRecord instance provides a validate method that performs basic checks (for instance, ensuring that a company name is present) before a record is saved into the database.
438
+
439
+ The snippet below shows how to wrap operations in try/catch blocks and use the validate method:
198
440
 
199
441
  ```typescript
200
- class GermanBusinessData {
201
- public async start() {
202
- await this.update();
442
+ import { OpenData } from '@fin.cx/opendata';
443
+
444
+ export const validateAndSaveRecord = async (openData: OpenData) => {
445
+ const record = new openData.CBusinessRecord();
446
+ record.data = {
447
+ name: "Validation Test Corp",
448
+ address: "Teststraße 99",
449
+ postalCode: "12345",
450
+ city: "Teststadt",
451
+ country: "Germany",
452
+ phone: "+49 123 456789",
453
+ email: "test@testcorp.de",
454
+ businessType: "AG",
455
+ registrationId: "District court Teststadt HRB 111111",
456
+ legalForm: "AG",
457
+ managingDirectors: ["Test Director"],
458
+ foundingDate: new Date().toISOString(),
459
+ capital: "1,000,000 EUR",
460
+ purpose: "Testing for data validation",
461
+ lastUpdate: new Date().toISOString()
462
+ };
463
+
464
+ try {
465
+ // Validate record data before saving.
466
+ await record.validate();
467
+ await record.save();
468
+ console.log("Record validated and saved successfully.");
469
+ } catch (error) {
470
+ console.error("Error validating or saving record:", error);
203
471
  }
472
+ };
473
+ ```
204
474
 
205
- public async update() {
206
- // Logic for updating business data using import streams and parsing JSON lines.
475
+ Using proper error handling ensures that the entire system remains reliable, and any data validation issues are caught early during development or in production.
476
+
477
+ ────────────────────────────────────────────
478
+ ### 7. Testing and Automated Workflows
479
+
480
+ To support continuous integration and adherence to best practices, the @fin.cx/opendata module includes tests written with @push.rocks/tapbundle. You should consider incorporating these tests in your development workflow. The tests verify all main functionalities including instance initialization, bulk data import, Handelsregister operations, and CRUD operations for BusinessRecords.
481
+
482
+ Below is an example of a simple test written in TypeScript using ESM that makes use of the module:
483
+
484
+ ```typescript
485
+ import { expect, tap } from '@push.rocks/tapbundle';
486
+ import { OpenData } from '@fin.cx/opendata';
487
+
488
+ let testOpenDataInstance: OpenData;
489
+
490
+ tap.test('Instance creation', async () => {
491
+ testOpenDataInstance = new OpenData();
492
+ expect(testOpenDataInstance).toBeInstanceOf(OpenData);
493
+ });
494
+
495
+ tap.test('Start instance', async () => {
496
+ await testOpenDataInstance.start();
497
+ });
498
+
499
+ tap.test('Perform bulk import', async () => {
500
+ await testOpenDataInstance.buildInitialDb();
501
+ });
502
+
503
+ tap.test('Stop instance', async () => {
504
+ await testOpenDataInstance.stop();
505
+ });
506
+
507
+ tap.start();
508
+ ```
509
+
510
+ This test code is designed to verify that the OpenData instance is successfully created, started, performs the critical bulk import operation, and is properly shutdown. Integration tests for the Handelsregister functionality follow a similar pattern and ensure that the browser automation routines and file download processes complete without errors.
511
+
512
+ ────────────────────────────────────────────
513
+ ### Comprehensive Example: Full Cycle from Initialization to Cleanup
514
+
515
+ To better illustrate how one might combine several aspects of the module in a production scenario, here's a comprehensive example that ties together initialization, CRUD operations, bulk processing, and Handelsregister interactions. This full-cycle example is written in TypeScript using ESM syntax and demonstrates how to build a production-grade data update and management pipeline.
516
+
517
+ ```typescript
518
+ import { OpenData } from '@fin.cx/opendata';
519
+
520
+ const runFullCyclePipeline = async () => {
521
+ const openData = new OpenData();
522
+
523
+ try {
524
+ // Initialize the module and connect to MongoDB
525
+ console.log('Initializing the OpenData module...');
526
+ await openData.start();
527
+
528
+ // Step 1: Bulk Import - Build the initial database from downloaded open data
529
+ console.log('Starting bulk data import from JSONL source...');
530
+ await openData.buildInitialDb();
531
+
532
+ // Step 2: Business Record Management - Create a sample business record
533
+ console.log('Creating a new business record...');
534
+ const sampleRecord = new openData.CBusinessRecord();
535
+ sampleRecord.data = {
536
+ name: "Sample Enterprise GmbH",
537
+ address: "Innovation Avenue 101",
538
+ postalCode: "80807",
539
+ city: "Munich",
540
+ country: "Germany",
541
+ phone: "+49 89 111222",
542
+ email: "info@sampleenterprise.de",
543
+ website: "https://sampleenterprise.de",
544
+ businessType: "GmbH",
545
+ registrationId: "District court Munich HRB 555555",
546
+ legalForm: "GmbH",
547
+ managingDirectors: ["Director A", "Director B"],
548
+ foundingDate: new Date("2015-06-15").toISOString(),
549
+ capital: "500,000 EUR",
550
+ purpose: "Holistic business solutions and data processing",
551
+ lastUpdate: new Date().toISOString()
552
+ };
553
+
554
+ await sampleRecord.save();
555
+ console.log('Sample business record created with id:', sampleRecord.id);
556
+
557
+ // Step 3: Retrieve business records for a specific location
558
+ console.log('Retrieving business records for Munich...');
559
+ const munichRecords = await openData.db
560
+ .collection('businessrecords')
561
+ .find({ city: "Munich" })
562
+ .toArray();
563
+ console.log(`Found ${munichRecords.length} records for Munich.`);
564
+
565
+ // Step 4: Update an existing record
566
+ if (munichRecords.length > 0) {
567
+ const recordToUpdateId = munichRecords[0].id;
568
+ console.log(`Updating business record with id: ${recordToUpdateId}`);
569
+ const recordToUpdate = await openData.CBusinessRecord.getInstance(recordToUpdateId);
570
+ if (recordToUpdate) {
571
+ recordToUpdate.data.phone = "+49 89 999888";
572
+ recordToUpdate.data.lastUpdate = new Date().toISOString();
573
+ await recordToUpdate.save();
574
+ console.log('Business record updated:', recordToUpdate);
575
+ }
576
+ }
577
+
578
+ // Step 5: Use Handelsregister to search for a specific company
579
+ console.log('Using Handelsregister to search for a specific company...');
580
+ const searchResults = await openData.handelsregister.searchCompany("Step Beyond GmbH", 20);
581
+ if (searchResults && searchResults.length > 0) {
582
+ const registrationData = searchResults[0].germanParsedRegistration;
583
+ console.log('Retrieved registration data:', registrationData);
584
+
585
+ // Step 6: Retrieve detailed info and download files for the specific company
586
+ console.log('Fetching detailed data for the identified company...');
587
+ const detailedInfo = await openData.handelsregister.getSpecificCompany(registrationData);
588
+ console.log('Detailed company data received:', detailedInfo);
589
+
590
+ // Optionally, save the downloaded files to a designated directory
591
+ for (const downloadedFile of detailedInfo.files) {
592
+ await downloadedFile.writeToDir('./output_files');
593
+ console.log(`Downloaded file saved at: ${downloadedFile.path}`);
594
+ }
595
+ } else {
596
+ console.log('No matching records found for detailed company data retrieval.');
597
+ }
598
+
599
+ // Step 7: Validate and save a record to demonstrate error handling and validation
600
+ console.log('Validating and saving a new test record...');
601
+ await validateAndSaveRecord(openData);
602
+
603
+ } catch (error) {
604
+ console.error('An error occurred during the full cycle pipeline operation:', error);
605
+ } finally {
606
+ // Final cleanup: Stop the OpenData module and release all resources
607
+ console.log('Finalizing: stopping the OpenData module...');
608
+ await openData.stop();
609
+ console.log('Pipeline completed and all resources have been cleaned up.');
207
610
  }
208
- }
611
+ };
612
+
613
+ runFullCyclePipeline();
209
614
  ```
210
615
 
211
- ### Ensuring Data Accuracy and Integrity
616
+ In this example, the entire processing cycle is constructed to mimic a realistic scenario. The pipeline:
617
+ • Starts by connecting to your database.
618
+ • Imports extensive JSONL open data.
619
+ • Creates, retrieves, updates, and deletes business records.
620
+ • Interacts with the Handelsregister for advanced company-specific operations.
621
+ • Implements robust error handling and validation routines, ensuring that each step is verifiable.
622
+ • Finally, ensures that resources such as MongoDB connections and headless browser sessions are responsibly closed.
623
+
624
+ ────────────────────────────────────────────
625
+ ### Final Thoughts on Module Integration
626
+
627
+ The @fin.cx/opendata library is designed to cater to a wide range of business data management needs. Whether you are an enterprise looking to integrate updated open data for decision-making or a developer looking to build data-rich applications with a focus on German companies, this library provides the tools and abstractions necessary to build robust solutions.
628
+
629
+ Every component—from the smart data management for business records to the advanced streaming and concurrent processing of JSONL files—is built with scalability and ease of use in mind. Integration with the Handelsregister via browser automation further extends its reach, providing dynamic access to official data sources in real-time.
630
+
631
+ As demonstrated in the examples above, each sub-component of the library is independent yet harmoniously integrated into a cohesive user experience. The use of ESM syntax throughout the module and the strict adherence to TypeScript definitions enhances reliability, maintainability, and the overall developer experience.
632
+
633
+ By following the usage scenarios provided in this documentation, you should now have a deep understanding of how to:
634
+ • Set up your environment and initialize the OpenData instance.
635
+ • Perform CRUD operations on business records.
636
+ • Efficiently process thousands of records from external JSONL sources.
637
+ • Integrate and automate Handelsregister interactions for detailed company data retrieval.
638
+ • Combine all building blocks into advanced automated workflows that support large-scale enterprise applications.
639
+
640
+ Feel free to explore, extend, and customize these examples to suit your project’s unique requirements. The library is designed with extensibility in mind, and additional utility functions or integrations can be added based on your needs.
641
+
642
+ We encourage you to integrate these practices into your development processes, run the provided tests, and contribute to further enhancements that can benefit the entire community of developers working on open data management systems.
212
643
 
213
- When working with business data, ensuring integrity and accuracy is crucial. Each record should be checked for validity before being saved or updated, minimizing inconsistencies. Moreover, robust error handling is essential in every step, from data retrieval to database operations, particularly when dealing with external data sources.
644
+ Happy coding and data integrating!
214
645
 
215
- The `@fin.cx/opendata` module provides an extensive toolset for accessing and managing business data, particularly for companies based in Germany. Its functionalities include creating, updating, retrieving, and deleting business records, as well as keeping them current with the latest open data releases. This makes it an invaluable asset for developers aiming to integrate open data seamlessly into their systems, ensuring robust data management capabilities within their applications.
216
646
 
217
- Happy exploring and integrating open data into your projects!
218
647
 
219
648
  ## License and Legal Information
220
649
 
@@ -233,4 +662,4 @@ Registered at District court Bremen HRB 35230 HB, Germany
233
662
 
234
663
  For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
235
664
 
236
- By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
665
+ By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
@@ -3,6 +3,6 @@
3
3
  */
4
4
  export const commitinfo = {
5
5
  name: '@fin.cx/opendata',
6
- version: '1.4.4',
7
- description: 'A TypeScript library for accessing, managing, and updating open business data, focused on German companies and integrating with MongoDB.'
6
+ version: '1.5.3',
7
+ description: 'A comprehensive TypeScript library that manages open business data for German companies by integrating MongoDB, processing JSONL bulk data, and automating browser interactions for Handelsregister data retrieval.'
8
8
  }
@@ -96,8 +96,9 @@ export class JsonlDataProcessor<T> {
96
96
  },
97
97
  finalFunction: async (streamToolsArg) => {
98
98
  console.log(`finished processing ${totalRecordsCounter} records.`);
99
- if (!nextRest) return;
100
- JSON.parse(nextRest);
99
+ if (nextRest) {
100
+ JSON.parse(nextRest);
101
+ };
101
102
  done.resolve();
102
103
  },
103
104
  })