@fin.cx/opendata 1.7.0 → 2.1.0

package/readme.md

@@ -4,6 +4,62 @@
 
 Access live stock prices, cryptocurrencies, forex, commodities AND comprehensive German company data - all through a single, unified API.
 
+## ⚠️ Breaking Changes
+
+### v2.1 - Enhanced Stock Market API (Current)
+
+**The stock market API has been significantly enhanced with a new unified request system.**
+
+**What Changed:**
+- New discriminated union request types (`IStockDataRequest`)
+- Enhanced `IStockPrice` interface with OHLCV data and metadata
+- New `getData()` method replaces legacy `getPrice()` and `getPrices()`
+- Historical data support with date ranges
+- Exchange filtering via MIC codes
+- Smart caching with data-type aware TTL
+
+**Migration:**
+```typescript
+// OLD (v2.0 and earlier)
+const price = await service.getPrice({ ticker: 'AAPL' });
+const prices = await service.getPrices({ tickers: ['AAPL', 'MSFT'] });
+
+// NEW (v2.1+) - Unified API
+const price = await service.getData({ type: 'current', ticker: 'AAPL' });
+const prices = await service.getData({ type: 'batch', tickers: ['AAPL', 'MSFT'] });
+
+// NEW - Historical data
+const history = await service.getData({
+  type: 'historical',
+  ticker: 'AAPL',
+  from: new Date('2024-01-01'),
+  to: new Date('2024-12-31')
+});
+
+// NEW - Exchange filtering
+const price = await service.getData({
+  type: 'current',
+  ticker: 'VOD',
+  exchange: 'XLON' // London Stock Exchange
+});
+```
+
+**Note:** Legacy `getPrice()` and `getPrices()` methods still work but are deprecated.
+
+### v2.0 - Directory Configuration
+
+**Directory paths are now MANDATORY when using German business data features.** The package no longer creates `.nogit/` directories automatically. You must explicitly configure all directory paths when instantiating `OpenData`:
+
+```typescript
+const openData = new OpenData({
+  nogitDir: '/path/to/your/data',
+  downloadDir: '/path/to/your/data/downloads',
+  germanBusinessDataDir: '/path/to/your/data/germanbusinessdata'
+});
+```
+
+This change enables the package to work in read-only filesystems (like Deno compiled binaries) and gives you full control over where data is stored.
+
 ## Installation
 
 ```bash
@@ -14,17 +70,17 @@ pnpm add @fin.cx/opendata
 
 ## Quick Start
 
-### 📈 Stock Market Data
+### 📈 Stock Market Data (v2.1+ Enhanced)
 
-Get market data with EOD (End-of-Day) pricing:
+Get comprehensive market data with EOD, historical, and OHLCV data:
 
 ```typescript
 import { StockPriceService, MarketstackProvider } from '@fin.cx/opendata';
 
-// Initialize the service with caching
+// Initialize the service with smart caching
 const stockService = new StockPriceService({
-  ttl: 60000,      // Cache for 1 minute
-  maxEntries: 1000 // Max cached symbols
+  ttl: 60000,       // Default cache TTL (historical data cached forever)
+  maxEntries: 10000  // Increased for historical data
 });
 
 // Register Marketstack provider with API key
@@ -33,18 +89,38 @@ stockService.register(new MarketstackProvider('YOUR_API_KEY'), {
   retryAttempts: 3
 });
 
-// Get single stock price
-const apple = await stockService.getPrice({ ticker: 'AAPL' });
+// Get current price (new unified API)
+const apple = await stockService.getData({ type: 'current', ticker: 'AAPL' });
 console.log(`Apple: $${apple.price} (${apple.changePercent.toFixed(2)}%)`);
+console.log(`OHLCV: O=${apple.open} H=${apple.high} L=${apple.low} V=${apple.volume}`);
+
+// Get historical data (1 year of daily prices)
+const history = await stockService.getData({
+  type: 'historical',
+  ticker: 'AAPL',
+  from: new Date('2024-01-01'),
+  to: new Date('2024-12-31'),
+  sort: 'DESC' // Newest first
+});
+console.log(`Fetched ${history.length} trading days`);
 
-// Get multiple prices at once (batch fetching)
-const prices = await stockService.getPrices({
-  tickers: ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA']
+// Exchange-specific data (London vs NYSE)
+const vodLondon = await stockService.getData({
+  type: 'current',
+  ticker: 'VOD',
+  exchange: 'XLON' // London Stock Exchange
+});
+
+const vodNYSE = await stockService.getData({
+  type: 'current',
+  ticker: 'VOD',
+  exchange: 'XNYS' // New York Stock Exchange
 });
 
-// 125,000+ tickers across 72+ exchanges worldwide
-const internationalStocks = await stockService.getPrices({
-  tickers: ['AAPL', 'VOD.LON', 'SAP.DEX', 'TM', 'BABA']
+// Batch current prices
+const prices = await stockService.getData({
+  type: 'batch',
+  tickers: ['AAPL', 'MSFT', 'GOOGL', 'AMZN', 'TSLA']
 });
 ```
 
@@ -54,8 +130,14 @@ Access comprehensive data on German companies:
 
 ```typescript
 import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
 
-const openData = new OpenData();
+// REQUIRED: Configure directory paths
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
 await openData.start();
 
 // Create a business record
@@ -80,15 +162,18 @@ await openData.buildInitialDb();
 
 ## Features
 
-### 🎯 Stock Market Module
+### 🎯 Stock Market Module (v2.1 Enhanced)
 
-- **Marketstack API** - End-of-Day (EOD) data for 125,000+ tickers across 72+ exchanges
-- **Stock prices** for stocks, ETFs, indices, and more
-- **Batch operations** - fetch 100+ symbols in one request
-- **Smart caching** - configurable TTL, automatic invalidation
-- **Extensible provider system** - easily add new data sources
-- **Retry logic** - configurable retry attempts and delays
-- **Type-safe** - full TypeScript support with detailed interfaces
+- **Historical Data** - Up to 15 years of daily EOD prices with automatic pagination
+- **Exchange Filtering** - Query specific exchanges via MIC codes (XNAS, XLON, XNYS, etc.)
+- **OHLCV Data** - Open, High, Low, Close, Volume for comprehensive analysis
+- **Smart Caching** - Data-type aware TTL (historical cached forever, EOD 24h, live 30s)
+- **Marketstack API** - 500,000+ tickers across 72+ exchanges worldwide
+- **Batch Operations** - Fetch 100+ symbols in one request
+- **Unified API** - Discriminated union types for type-safe requests
+- **Extensible Providers** - Easy to add new data sources
+- **Retry Logic** - Configurable attempts and delays
+- **Type-Safe** - Full TypeScript support with detailed interfaces
 
 ### 🇩🇪 German Business Intelligence
 
@@ -100,6 +185,151 @@ await openData.buildInitialDb();
 
 ## Advanced Examples
 
+### Phase 1: Historical Data Analysis
+
+Fetch and analyze historical price data:
+
+```typescript
+// Get 1 year of historical data
+const history = await stockService.getData({
+  type: 'historical',
+  ticker: 'AAPL',
+  from: new Date('2024-01-01'),
+  to: new Date('2024-12-31'),
+  sort: 'DESC' // Newest first (default)
+});
+
+// Calculate statistics
+const prices = history.map(p => p.price);
+const high52Week = Math.max(...prices);
+const low52Week = Math.min(...prices);
+const avgPrice = prices.reduce((a, b) => a + b) / prices.length;
+
+console.log(`52-Week Analysis for AAPL:`);
+console.log(`High:    $${high52Week.toFixed(2)}`);
+console.log(`Low:     $${low52Week.toFixed(2)}`);
+console.log(`Average: $${avgPrice.toFixed(2)}`);
+console.log(`Days:    ${history.length}`);
+
+// Calculate daily returns
+for (let i = 0; i < history.length - 1; i++) {
+  const todayPrice = history[i].price;
+  const yesterdayPrice = history[i + 1].price;
+  const dailyReturn = ((todayPrice - yesterdayPrice) / yesterdayPrice) * 100;
+  console.log(`${history[i].timestamp.toISOString().split('T')[0]}: ${dailyReturn.toFixed(2)}%`);
+}
+```
+
+### Phase 1: Exchange-Specific Trading
+
+Compare prices across different exchanges:
+
+```typescript
+// Vodafone trades on both London and NYSE
+const exchanges = [
+  { mic: 'XLON', name: 'London Stock Exchange' },
+  { mic: 'XNYS', name: 'New York Stock Exchange' }
+];
+
+for (const exchange of exchanges) {
+  try {
+    const price = await stockService.getData({
+      type: 'current',
+      ticker: 'VOD',
+      exchange: exchange.mic
+    });
+
+    console.log(`${exchange.name}:`);
+    console.log(`  Price: ${price.price} ${price.currency}`);
+    console.log(`  Volume: ${price.volume?.toLocaleString()}`);
+    console.log(`  Exchange: ${price.exchangeName}`);
+  } catch (error) {
+    console.log(`${exchange.name}: Not available`);
+  }
+}
+```
+
+### Phase 1: OHLCV Technical Analysis
+
+Use OHLCV data for technical indicators:
+
+```typescript
+const history = await stockService.getData({
+  type: 'historical',
+  ticker: 'TSLA',
+  from: new Date('2024-11-01'),
+  to: new Date('2024-11-30')
+});
+
+// Calculate daily trading range
+for (const day of history) {
+  const range = day.high - day.low;
+  const rangePercent = (range / day.low) * 100;
+
+  console.log(`${day.timestamp.toISOString().split('T')[0]}:`);
+  console.log(`  Open:   $${day.open}`);
+  console.log(`  High:   $${day.high}`);
+  console.log(`  Low:    $${day.low}`);
+  console.log(`  Close:  $${day.price}`);
+  console.log(`  Volume: ${day.volume?.toLocaleString()}`);
+  console.log(`  Range:  $${range.toFixed(2)} (${rangePercent.toFixed(2)}%)`);
+}
+
+// Calculate Simple Moving Average (SMA)
+const calculateSMA = (data: IStockPrice[], period: number) => {
+  const sma: number[] = [];
+  for (let i = period - 1; i < data.length; i++) {
+    const sum = data.slice(i - period + 1, i + 1)
+      .reduce((acc, p) => acc + p.price, 0);
+    sma.push(sum / period);
+  }
+  return sma;
+};
+
+const sma20 = calculateSMA(history, 20);
+const sma50 = calculateSMA(history, 50);
+
+console.log(`20-day SMA: $${sma20[0].toFixed(2)}`);
+console.log(`50-day SMA: $${sma50[0].toFixed(2)}`);
+```
+
+### Phase 1: Smart Caching Performance
+
+Leverage smart caching for efficiency:
+
+```typescript
+// Historical data is cached FOREVER (never changes)
+console.time('First historical fetch');
+const history1 = await stockService.getData({
+  type: 'historical',
+  ticker: 'AAPL',
+  from: new Date('2024-01-01'),
+  to: new Date('2024-12-31')
+});
+console.timeEnd('First historical fetch');
+// Output: First historical fetch: 2341ms
+
+console.time('Second historical fetch (cached)');
+const history2 = await stockService.getData({
+  type: 'historical',
+  ticker: 'AAPL',
+  from: new Date('2024-01-01'),
+  to: new Date('2024-12-31')
+});
+console.timeEnd('Second historical fetch (cached)');
+// Output: Second historical fetch (cached): 2ms (1000x faster!)
+
+// EOD data cached for 24 hours
+const currentPrice = await stockService.getData({
+  type: 'current',
+  ticker: 'MSFT'
+});
+// Subsequent calls within 24h served from cache
+
+// Cache statistics
+console.log(`Cache size: ${stockService['cache'].size} entries`);
+```
+
 ### Market Dashboard
 
 Create an EOD market overview:
@@ -159,6 +389,17 @@ console.log('Marketstack Stats:', {
 Automate German company data retrieval:
 
 ```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+// Configure paths first
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+await openData.start();
+
 // Search for a company
 const results = await openData.handelsregister.searchCompany("Siemens AG");
 
@@ -182,6 +423,21 @@ for (const file of details.files) {
 Merge financial and business data:
 
 ```typescript
+import { OpenData, StockPriceService, MarketstackProvider } from '@fin.cx/opendata';
+import * as path from 'path';
+
+// Configure OpenData with paths
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+await openData.start();
+
+// Setup stock service
+const stockService = new StockPriceService({ ttl: 60000, maxEntries: 1000 });
+stockService.register(new MarketstackProvider('YOUR_API_KEY'));
+
 // Find all public German companies (AG)
 const publicCompanies = await openData.db
   .collection('businessrecords')
@@ -212,6 +468,48 @@ for (const company of publicCompanies) {
 
 ## Configuration
 
+### Directory Configuration (Required for German Business Data)
+
+**All directory paths are mandatory when using `OpenData`.** Here are examples for different environments:
+
+#### Development Environment
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+const openData = new OpenData({
+  nogitDir: path.join(process.cwd(), '.nogit'),
+  downloadDir: path.join(process.cwd(), '.nogit', 'downloads'),
+  germanBusinessDataDir: path.join(process.cwd(), '.nogit', 'germanbusinessdata')
+});
+```
+
+#### Production Environment
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+import * as path from 'path';
+
+const openData = new OpenData({
+  nogitDir: '/var/lib/myapp/data',
+  downloadDir: '/var/lib/myapp/data/downloads',
+  germanBusinessDataDir: '/var/lib/myapp/data/germanbusinessdata'
+});
+```
+
+#### Deno Compiled Binaries (or other read-only filesystems)
+```typescript
+import { OpenData } from '@fin.cx/opendata';
+
+// Use OS temp directory or user data directory
+const dataDir = Deno.env.get('HOME') + '/.myapp/data';
+
+const openData = new OpenData({
+  nogitDir: dataDir,
+  downloadDir: dataDir + '/downloads',
+  germanBusinessDataDir: dataDir + '/germanbusinessdata'
+});
+```
+
 ### Stock Service Options
 
 ```typescript

package/ts/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: '@fin.cx/opendata',
-  version: '1.7.0',
+  version: '2.1.0',
   description: 'A comprehensive TypeScript library for accessing business data and real-time financial information. Features include German company data management with MongoDB integration, JSONL bulk processing, automated Handelsregister interactions, and real-time stock market data from multiple providers.'
 }

package/ts/classes.handelsregister.ts

@@ -1,7 +1,6 @@
 import type { BusinessRecord } from './classes.businessrecord.js';
 import type { OpenData } from './classes.main.opendata.js';
 import * as plugins from './plugins.js';
-import * as paths from './paths.js';
 
 /**
  * the HandlesRegister exposed as a class
@@ -9,13 +8,16 @@ import * as paths from './paths.js';
 export class HandelsRegister {
   private openDataRef: OpenData;
   private asyncExecutionStack = new plugins.lik.AsyncExecutionStack();
-  private uniqueDowloadFolder = plugins.path.join(paths.downloadDir, plugins.smartunique.uniSimple()); 
+  private downloadDir: string;
+  private uniqueDowloadFolder: string;
 
   // Puppeteer wrapper instance
   public smartbrowserInstance = new plugins.smartbrowser.SmartBrowser();
 
-  constructor(openDataRef: OpenData) {
+  constructor(openDataRef: OpenData, downloadDirArg: string) {
     this.openDataRef = openDataRef;
+    this.downloadDir = downloadDirArg;
+    this.uniqueDowloadFolder = plugins.path.join(this.downloadDir, plugins.smartunique.uniSimple());
   }
 
   public async start() {
@@ -76,7 +78,7 @@ export class HandelsRegister {
         timeout: 30000,
       })
       .catch(async (err) => {
-        await pageArg.screenshot({ path: paths.downloadDir + '/error.png' });
+        await pageArg.screenshot({ path: this.downloadDir + '/error.png' });
         throw err;
       });
 

package/ts/classes.jsonldata.ts

@@ -1,5 +1,4 @@
 import * as plugins from './plugins.js';
-import * as paths from './paths.js';
 import type { OpenData } from './classes.main.opendata.js';
 
 export type SeedEntryType = {
@@ -41,8 +40,11 @@ export type SeedEntryType = {
 };
 
 export class JsonlDataProcessor<T> {
+  private germanBusinessDataDir: string;
   public forEachFunction: (entryArg: T) => Promise<void>;
-  constructor(forEachFunctionArg: typeof this.forEachFunction) {
+
+  constructor(germanBusinessDataDirArg: string, forEachFunctionArg: typeof this.forEachFunction) {
+    this.germanBusinessDataDir = germanBusinessDataDirArg;
     this.forEachFunction = forEachFunctionArg;
   }
 
@@ -51,9 +53,9 @@ export class JsonlDataProcessor<T> {
     dataUrlArg = 'https://daten.offeneregister.de/de_companies_ocdata.jsonl.bz2'
   ) {
     const done = plugins.smartpromise.defer();
-    const dataExists = await plugins.smartfile.fs.isDirectory(paths.germanBusinessDataDir);
+    const dataExists = await plugins.smartfile.fs.isDirectory(this.germanBusinessDataDir);
     if (!dataExists) {
-      await plugins.smartfile.fs.ensureDir(paths.germanBusinessDataDir);
+      await plugins.smartfile.fs.ensureDir(this.germanBusinessDataDir);
     } else {
     }
 

package/ts/classes.main.opendata.ts

@@ -4,16 +4,39 @@ import { JsonlDataProcessor, type SeedEntryType } from './classes.jsonldata.js';
 import * as paths from './paths.js';
 import * as plugins from './plugins.js';
 
+export interface IOpenDataConfig {
+  downloadDir: string;
+  germanBusinessDataDir: string;
+  nogitDir: string;
+}
+
 export class OpenData {
   public db: plugins.smartdata.SmartdataDb;
-  private serviceQenv = new plugins.qenv.Qenv(paths.packageDir, paths.nogitDir);
+  private serviceQenv: plugins.qenv.Qenv;
+  private config: IOpenDataConfig;
 
   public jsonLDataProcessor: JsonlDataProcessor<SeedEntryType>;
   public handelsregister: HandelsRegister;
 
   public CBusinessRecord = plugins.smartdata.setDefaultManagerForDoc(this, BusinessRecord);
 
+  constructor(configArg: IOpenDataConfig) {
+    if (!configArg) {
+      throw new Error('@fin.cx/opendata: Configuration is required. You must provide downloadDir, germanBusinessDataDir, and nogitDir paths.');
+    }
+    if (!configArg.downloadDir || !configArg.germanBusinessDataDir || !configArg.nogitDir) {
+      throw new Error('@fin.cx/opendata: All directory paths are required (downloadDir, germanBusinessDataDir, nogitDir).');
+    }
+    this.config = configArg;
+    this.serviceQenv = new plugins.qenv.Qenv(paths.packageDir, this.config.nogitDir);
+  }
+
   public async start() {
+    // Ensure configured directories exist
+    await plugins.smartfile.fs.ensureDir(this.config.nogitDir);
+    await plugins.smartfile.fs.ensureDir(this.config.downloadDir);
+    await plugins.smartfile.fs.ensureDir(this.config.germanBusinessDataDir);
+
     this.db = new plugins.smartdata.SmartdataDb({
       mongoDbUrl: await this.serviceQenv.getEnvVarOnDemand('MONGODB_URL'),
       mongoDbName: await this.serviceQenv.getEnvVarOnDemand('MONGODB_NAME'),
@@ -21,18 +44,21 @@ export class OpenData {
       mongoDbPass: await this.serviceQenv.getEnvVarOnDemand('MONGODB_PASS'),
     });
     await this.db.init();
-    this.jsonLDataProcessor = new JsonlDataProcessor(async (entryArg) => {
-      const businessRecord = new this.CBusinessRecord();
-      businessRecord.id = await this.CBusinessRecord.getNewId();
-      businessRecord.data.name = entryArg.name;
-      businessRecord.data.germanParsedRegistration = {
-        court: entryArg.all_attributes.registered_office,
-        number: entryArg.all_attributes._registerNummer,
-        type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
-      };
-      await businessRecord.save();
-    });
-    this.handelsregister = new HandelsRegister(this);
+    this.jsonLDataProcessor = new JsonlDataProcessor(
+      this.config.germanBusinessDataDir,
+      async (entryArg) => {
+        const businessRecord = new this.CBusinessRecord();
+        businessRecord.id = await this.CBusinessRecord.getNewId();
+        businessRecord.data.name = entryArg.name;
+        businessRecord.data.germanParsedRegistration = {
+          court: entryArg.all_attributes.registered_office,
+          number: entryArg.all_attributes._registerNummer,
+          type: entryArg.all_attributes._registerArt as 'HRA' | 'HRB',
+        };
+        await businessRecord.save();
+      }
+    );
+    this.handelsregister = new HandelsRegister(this, this.config.downloadDir);
     await this.handelsregister.start();
   }
 

package/ts/paths.ts

@@ -3,13 +3,4 @@ import * as plugins from './plugins.js';
 export const packageDir = plugins.path.join(
   plugins.smartpath.get.dirnameFromImportMetaUrl(import.meta.url),
   '../'
-);
-
-export const nogitDir = plugins.path.join(packageDir, './.nogit/');
-plugins.smartfile.fs.ensureDirSync(nogitDir);
-
-export const downloadDir = plugins.path.join(nogitDir, 'downloads');
-plugins.smartfile.fs.ensureDirSync(downloadDir);
-
-
-export const germanBusinessDataDir = plugins.path.join(nogitDir, 'germanbusinessdata');
+);