@fin.cx/opendata 1.6.1 → 2.0.0

package/readme.md

@@ -1,41 +1,64 @@
 # @fin.cx/opendata
+
 🚀 **Real-time financial data and German business intelligence in one powerful TypeScript library**
 
 Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API.
 
+## ⚠️ Breaking Change in v2.0
+
+**Directory paths are now MANDATORY when using German business data features.** The package no longer creates `.nogit/` directories automatically. You must explicitly configure all directory paths when instantiating `OpenData`:
+
+```typescript
+const openData = new OpenData({
+  nogitDir: '/path/to/your/data',
+  downloadDir: '/path/to/your/data/downloads',
+  germanBusinessDataDir: '/path/to/your/data/germanbusinessdata'
+});
+```
+
+This change enables the package to work in read-only filesystems (like Deno compiled binaries) and gives you full control over where data is stored.
+
 ## Installation
 
 ```bash
 npm install @fin.cx/opendata
 # or
-yarn add @fin.cx/opendata
+pnpm add @fin.cx/opendata
 ```
 
 ## Quick Start
 
-### 📈 Real-Time Stock Data
+### 📈 Stock Market Data
 
-Get live market data in seconds:
+Get market data with EOD (End-of-Day) pricing:
 
 ```typescript
-import { StockPriceService, YahooFinanceProvider } from '@fin.cx/opendata';
+import { StockPriceService, MarketstackProvider } from '@fin.cx/opendata';
+
+// Initialize the service with caching
+const stockService = new StockPriceService({
+  ttl: 60000,      // Cache for 1 minute
+  maxEntries: 1000 // Max cached symbols
+});
 
-// Initialize the service
-const stockService = new StockPriceService();
-stockService.register(new YahooFinanceProvider());
+// Register Marketstack provider with API key
+stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
+  priority: 100,
+  retryAttempts: 3
+});
 
 // Get single stock price
 const apple = await stockService.getPrice({ ticker: 'AAPL' });
 console.log(`Apple: $${apple.price} (${apple.changePercent.toFixed(2)}%)`);
 
-// Get multiple prices at once
-const prices = await stockService.getPrices({ 
-  tickers: ['AAPL', 'MSFT', 'GOOGL', 'BTC-USD', 'ETH-USD'] 
+// Get multiple prices at once (batch fetching)
+const prices = await stockService.getPrices({
+  tickers: ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA']
 });
 
-// Market indices, crypto, forex, commodities - all supported!
-const marketData = await stockService.getPrices({
-  tickers: ['^GSPC', '^DJI', 'BTC-USD', 'EURUSD=X', 'GC=F']
+// 125,000+ tickers across 72+ exchanges worldwide
+const internationalStocks = await stockService.getPrices({
+  tickers: ['AAPL', 'VOD.LON', 'SAP.DEX', 'TM', 'BABA']
 });
 ```
 
@@ -45,8 +68,14 @@ Access comprehensive data on German companies:
 
 ```typescript
 import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
 
-const openData = new OpenData();
+// REQUIRED: Configure directory paths
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
 await openData.start();
 
 // Create a business record
@@ -73,11 +102,12 @@ await openData.buildInitialDb();
 
 ### 🎯 Stock Market Module
 
-- **Real-time prices** for stocks, ETFs, indices, crypto, forex, and commodities
+- **Marketstack API** - End-of-Day (EOD) data for 125,000+ tickers across 72+ exchanges
+- **Stock prices** for stocks, ETFs, indices, and more
 - **Batch operations** - fetch 100+ symbols in one request
 - **Smart caching** - configurable TTL, automatic invalidation
-- **Provider system** - easily extensible for new data sources
-- **Automatic retries** and fallback mechanisms
+- **Extensible provider system** - easily add new data sources
+- **Retry logic** - configurable retry attempts and delays
 - **Type-safe** - full TypeScript support with detailed interfaces
 
 ### 🇩🇪 German Business Intelligence
@@ -92,29 +122,24 @@ await openData.buildInitialDb();
 
 ### Market Dashboard
 
-Create a real-time market overview:
+Create an EOD market overview:
 
 ```typescript
 const indicators = [
   // Indices
   { ticker: '^GSPC', name: 'S&P 500' },
-  { ticker: '^IXIC', name: 'NASDAQ' },
-  
+  { ticker: '^DJI', name: 'DOW Jones' },
+
   // Tech Giants
   { ticker: 'AAPL', name: 'Apple' },
   { ticker: 'MSFT', name: 'Microsoft' },
-  
-  // Crypto
-  { ticker: 'BTC-USD', name: 'Bitcoin' },
-  { ticker: 'ETH-USD', name: 'Ethereum' },
-  
-  // Commodities
-  { ticker: 'GC=F', name: 'Gold' },
-  { ticker: 'CL=F', name: 'Oil' }
+  { ticker: 'GOOGL', name: 'Alphabet' },
+  { ticker: 'AMZN', name: 'Amazon' },
+  { ticker: 'TSLA', name: 'Tesla' }
 ];
 
-const prices = await stockService.getPrices({ 
-  tickers: indicators.map(i => i.ticker) 
+const prices = await stockService.getPrices({
+  tickers: indicators.map(i => i.ticker)
 });
 
 // Display with color-coded changes
@@ -122,7 +147,7 @@ prices.forEach(price => {
   const indicator = indicators.find(i => i.ticker === price.ticker);
   const arrow = price.change >= 0 ? '↑' : '↓';
   const color = price.change >= 0 ? '\x1b[32m' : '\x1b[31m';
-  
+
   console.log(
     `${indicator.name.padEnd(15)} ${price.price.toFixed(2).padStart(10)} ` +
     `${color}${arrow} ${price.changePercent.toFixed(2)}%\x1b[0m`
@@ -130,11 +155,41 @@ prices.forEach(price => {
 });
 ```
 
+### Provider Health and Statistics
+
+Monitor your provider health and track usage:
+
+```typescript
+// Check provider health
+const health = await stockService.checkProvidersHealth();
+console.log(`Marketstack: ${health.get('Marketstack') ? '✅' : '❌'}`);
+
+// Get provider statistics
+const stats = stockService.getProviderStats();
+const marketstackStats = stats.get('Marketstack');
+console.log('Marketstack Stats:', {
+  successCount: marketstackStats.successCount,
+  errorCount: marketstackStats.errorCount,
+  lastError: marketstackStats.lastError
+});
+```
+
 ### Handelsregister Integration
 
 Automate German company data retrieval:
 
 ```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+// Configure paths first
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+await openData.start();
+
 // Search for a company
 const results = await openData.handelsregister.searchCompany("Siemens AG");
 
@@ -158,6 +213,21 @@ for (const file of details.files) {
 Merge financial and business data:
 
 ```typescript
+import { OpenData, StockPriceService, MarketstackProvider } from '@fin.cx/opendata';
+import * as path from 'path';
+
+// Configure OpenData with paths
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+await openData.start();
+
+// Setup stock service
+const stockService = new StockPriceService({ ttl: 60000, maxEntries: 1000 });
+stockService.register(new MarketstackProvider('YOUR_API_KEY'));
+
 // Find all public German companies (AG)
 const publicCompanies = await openData.db
   .collection('businessrecords')
@@ -169,15 +239,15 @@ for (const company of publicCompanies) {
   try {
     // Map company to ticker (custom logic needed)
     const ticker = mapCompanyToTicker(company.data.name);
-    
+
     if (ticker) {
       const stock = await stockService.getPrice({ ticker });
-      
+
       // Add financial metrics
       company.data.stockPrice = stock.price;
       company.data.marketCap = stock.price * getSharesOutstanding(ticker);
       company.data.priceChange = stock.changePercent;
-      
+
       await company.save();
     }
   } catch (error) {
@@ -188,6 +258,48 @@ for (const company of publicCompanies) {
 
 ## Configuration
 
+### Directory Configuration (Required for German Business Data)
+
+**All directory paths are mandatory when using `OpenData`.** Here are examples for different environments:
+
+#### Development Environment
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+```
+
+#### Production Environment
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+const openData = new OpenData({
+  nogitDir: '/var/lib/myapp/data',
+  downloadDir: '/var/lib/myapp/data/downloads',
+  germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata'
+});
+```
+
+#### Deno Compiled Binaries (or other read-only filesystems)
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+
+// Use OS temp directory or user data directory
+const dataDir = Deno.env.get('HOME') + '/.myapp/data';
+
+const openData = new OpenData({
+  nogitDir: dataDir,
+  downloadDir: dataDir + '/downloads',
+  germanBusinessDataDir: dataDir + '/germanbusinessdata'
+});
+```
+
 ### Stock Service Options
 
 ```typescript
@@ -196,8 +308,8 @@ const stockService = new StockPriceService({
   maxEntries: 1000    // Max cached symbols
 });
 
-// Provider configuration
-stockService.register(new YahooFinanceProvider(), {
+// Marketstack - EOD data, requires API key
+stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
   enabled: true,
   priority: 100,
   timeout: 10000,
@@ -208,7 +320,7 @@ stockService.register(new YahooFinanceProvider(), {
 
 ### MongoDB Setup
 
-Set environment variables:
+Set environment variables for German business data:
 
 ```env
 MONGODB_URL=mongodb://localhost:27017
@@ -217,6 +329,14 @@ MONGODB_USER=myuser
 MONGODB_PASS=mypass
 ```
 
+### Marketstack API Key
+
+Get your free API key at [marketstack.com](https://marketstack.com) and set it in your environment:
+
+```env
+MARKETSTACK_COM_TOKEN=your_api_key_here
+```
+
 ## API Reference
 
 ### Stock Types
@@ -240,10 +360,21 @@ interface IStockPrice {
 ### Key Methods
 
 **StockPriceService**
-- `getPrice(request)` - Single stock price
-- `getPrices(request)` - Batch prices
-- `register(provider)` - Add data provider
+- `getPrice(request)` - Single stock price with automatic provider selection
+- `getPrices(request)` - Batch prices (100+ symbols in one request)
+- `register(provider, config)` - Add data provider with priority and retry config
+- `checkProvidersHealth()` - Test all providers and return health status
+- `getProviderStats()` - Get success/error statistics for each provider
 - `clearCache()` - Clear price cache
+- `setCacheTTL(ttl)` - Update cache TTL dynamically
+
+**MarketstackProvider**
+- ✅ End-of-Day (EOD) data
+- ✅ 125,000+ tickers across 72+ exchanges worldwide
+- ✅ Batch fetching support (multiple symbols in one request)
+- ✅ Comprehensive data: open, high, low, close, volume, splits, dividends
+- ⚠️ Requires API key (free tier: 100 requests/month)
+- ⚠️ EOD data only (not real-time)
 
 **OpenData**
 - `start()` - Initialize MongoDB connection
@@ -251,31 +382,48 @@ interface IStockPrice {
 - `CBusinessRecord` - Business record class
 - `handelsregister` - Registry automation
 
-## Performance
-
-- **Batch fetching**: Get 100+ prices in <500ms
-- **Caching**: Instant repeated queries
-- **Concurrent processing**: Handle 1000+ records/second
-- **Streaming**: Process GB-sized datasets without memory issues
-
-## Extensibility
+## Provider Architecture
 
-The provider architecture makes it easy to add new data sources:
+The library uses a flexible provider system that makes it easy to add new data sources:
 
 ```typescript
 class MyCustomProvider implements IStockProvider {
   name = 'My Provider';
-  
+  priority = 50;
+  requiresAuth = true;
+
   async fetchPrice(request: IStockQuoteRequest): Promise<IStockPrice> {
     // Your implementation
   }
-  
-  // ... other required methods
+
+  async fetchPrices(request: IStockBatchQuoteRequest): Promise<IStockPrice[]> {
+    // Batch implementation
+  }
+
+  async isAvailable(): Promise<boolean> {
+    // Health check
+  }
+
+  supportsMarket(market: string): boolean {
+    // Market validation
+  }
+
+  supportsTicker(ticker: string): boolean {
+    // Ticker validation
+  }
 }
 
 stockService.register(new MyCustomProvider());
 ```
 
+## Performance
+
+- **Batch fetching**: Get 100+ EOD prices in one API request
+- **Smart caching**: Instant repeated queries with configurable TTL
+- **Rate limit aware**: Automatic retry logic for API limits
+- **Concurrent processing**: Handle 1000+ business records/second
+- **Streaming**: Process GB-sized datasets without memory issues
+
 ## Testing
 
 Run the comprehensive test suite:
@@ -284,19 +432,21 @@ Run the comprehensive test suite:
 npm test
 ```
 
-View live market data:
+Test stock provider:
 
 ```bash
-npm test -- --grep "market indicators"
+npx tstest test/test.marketstack.node.ts --verbose
 ```
 
-## Contributing
+Test German business data:
 
-We welcome contributions! Please see our contributing guidelines for details.
+```bash
+npx tstest test/test.handelsregister.ts --verbose
+```
 
 ## License and Legal Information
 
-This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository. 
+This repository contains open-source code that is licensed under the MIT License. A copy of the MIT License can be found in the [license](license) file within this repository.
 
 **Please note:** The MIT License does not grant permission to use the trade names, trademarks, service marks, or product names of the project, except as required for reasonable and customary use in describing the origin of the work and reproducing the content of the NOTICE file.
 
@@ -306,9 +456,9 @@ This project is owned and maintained by Task Venture Capital GmbH. The names and
 
 ### Company Information
 
-Task Venture Capital GmbH  
+Task Venture Capital GmbH
 Registered at District court Bremen HRB 35230 HB, Germany
 
 For any legal inquiries or if you require further information, please contact us via email at hello@task.vc.
 
-By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.
+By using this repository, you acknowledge that you have read this section, agree to comply with its terms, and understand that the licensing of the code does not imply endorsement by Task Venture Capital GmbH of any derivative works.

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@fin.cx/opendata',
-  version: '1.6.1',
+  version: '2.0.0',
   description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
 }

package/ts/classes.handelsregister.ts

@@ -1,7 +1,6 @@
 import type { BusinessRecord } from './classes.businessrecord.js';
 import type { OpenData } from './classes.main.opendata.js';
 import * as plugins from './plugins.js';
-import * as paths from './paths.js';
 
 /**
  * the HandlesRegister exposed as a class
@@ -9,13 +8,16 @@ import * as paths from './paths.js';
 export class HandelsRegister {
   private openDataRef: OpenData;
   private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
-  private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple()); 
+  private downloadDir: string;
+  private uniqueDowloadFolder: string;
 
   // Puppeteer wrapper instance
   public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
 
-  constructor(openDataRef: OpenData) {
+  constructor(openDataRef: OpenData, downloadDirArg: string) {
     this.openDataRef = openDataRef;
+    this.downloadDir = downloadDirArg;
+    this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple());
   }
 
   public async start() {
@@ -76,7 +78,7 @@ export class HandelsRegister {
         timeout: 30000,
       })
       .catch(async (err) => {
-        await pageArg.screenshot({ path: paths.downloadDir + '/error.png' });
+        await pageArg.screenshot({ path: this.downloadDir + '/error.png' });
         throw err;
       });
 

package/ts/classes.jsonldata.ts

@@ -1,5 +1,4 @@
 import * as plugins from './plugins.js';
-import * as paths from './paths.js';
 import type { OpenData } from './classes.main.opendata.js';
 
 export type SeedEntryType = {
@@ -41,8 +40,11 @@ export type SeedEntryType = {
 };
 
 export class JsonlDataProcessor<T> {
+  private germanBusinessDataDir: string;
   public forEachFunction: (entryArg: T) => Promise<void>;
-  constructor(forEachFunctionArg: typeof this.forEachFunction) {
+
+  constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) {
+    this.germanBusinessDataDir = germanBusinessDataDirArg;
     this.forEachFunction = forEachFunctionArg;
   }
 
@@ -51,9 +53,9 @@ export class JsonlDataProcessor<T> {
     dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
   ) {
     const done = plugins.smartpromise.defer();
-    const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
+    const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir);
     if (!dataExists) {
-      await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
+      await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir);
     } else {
     }
 

package/ts/classes.main.opendata.ts

@@ -4,16 +4,39 @@ import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
 import * as paths from './paths.js';
 import * as plugins from './plugins.js';
 
+export interface IOpenDataConfig {
+  downloadDir: string;
+  germanBusinessDataDir: string;
+  nogitDir: string;
+}
+
 export class OpenData {
   public db: plugins.smartdata.SmartdataDb;
-  private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
+  private serviceQenv: plugins.qenv.Qenv;
+  private config: IOpenDataConfig;
 
   public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
   public handelsregister: HandelsRegister;
 
   public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
 
+  constructor(configArg: IOpenDataConfig) {
+    if (!configArg) {
+      throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.');
+    }
+    if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) {
+      throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
+    }
+    this.config = configArg;
+    this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir);
+  }
+
   public async start() {
+    // Ensure configured directories exist
+    await plugins.smartfile.fs.ensureDir(this.config.nogitDir);
+    await plugins.smartfile.fs.ensureDir(this.config.downloadDir);
+    await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir);
+
     this.db = new plugins.smartdata.SmartdataDb({
       mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
       mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
@@ -21,18 +44,21 @@ export class OpenData {
       mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
     });
     await this.db.init();
-    this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => {
-      const businessRecord = new this.CBusinessRecord();
-      businessRecord.id = await this.CBusinessRecord.getNewId();
-      businessRecord.data.name = entryArg.name;
-      businessRecord.data.germanParsedRegistration = {
-        court: entryArg.all_attributes.registered_office,
-        number: entryArg.all_attributes._registerNummer,
-        type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
-      };
-      await businessRecord.save();
-    });
-    this.handelsregister = new HandelsRegister(this);
+    this.jsonLDataProcessor = new JsonlDataProcessor(
+      this.config.germanBusinessDataDir,
+      async (entryArg) => {
+        const businessRecord = new this.CBusinessRecord();
+        businessRecord.id = await this.CBusinessRecord.getNewId();
+        businessRecord.data.name = entryArg.name;
+        businessRecord.data.germanParsedRegistration = {
+          court: entryArg.all_attributes.registered_office,
+          number: entryArg.all_attributes._registerNummer,
+          type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
+        };
+        await businessRecord.save();
+      }
+    );
+    this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
     await this.handelsregister.start();
   }
 

package/ts/paths.ts

@@ -3,13 +3,4 @@ import * as plugins from './plugins.js';
 export const packageDir = plugins.path.join(
   plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
   '../'
-);
-
-export const nogitDir = plugins.path.join(packageDir, './.nogit/');
-plugins.smartfile.fs.ensureDirSync(nogitDir);
-
-export const downloadDir = plugins.path.join(nogitDir, 'downloads');
-plugins.smartfile.fs.ensureDirSync(downloadDir);
-
-
-export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');
+);

package/ts/stocks/index.ts

@@ -6,4 +6,5 @@ export * from './interfaces/provider.js';
 export * from './classes.stockservice.js';
 
 // Export providers
-export * from './providers/provider.yahoo.js';
+export * from './providers/provider.yahoo.js';
+export * from './providers/provider.marketstack.js';