gdelt-ts-client 1.1.0 → 1.2.1

package/README.md

@@ -7,6 +7,7 @@ A comprehensive TypeScript client for the GDELT API that provides strongly-typed
 - **Complete API Coverage**: Access all GDELT API endpoints including articles, images, timelines, tone analysis, and more
 - **Type Safety**: Comprehensive TypeScript definitions for all parameters, responses, and configuration options
 - **Query Building**: Fluent API for constructing complex search queries with type safety
+- **Article Content**: Fetch and parse the full content of articles returned by GDELT API calls
 - **Validation**: Built-in parameter validation with helpful error messages
 - **Retry Logic**: Configurable retry mechanism for handling transient network errors
 - **Response Validation**: Type guards and validation for ensuring data integrity
@@ -167,12 +168,88 @@ const query = client.query()
 const articles = await client.getArticles(query);
 ```
 
+#### Advanced Query Builder Capabilities
+
+The query builder provides a comprehensive set of methods for constructing complex queries:
+
+```typescript
+// Basic query operations
+const query = client.query()
+  .search('climate')                                // Simple search term
+  .phrase('global warming')                         // Exact phrase match
+  .anyOf('policy', 'agreement', 'treaty')           // Match any of these terms
+  .allOf('emissions', 'reduction', 'targets')       // Match all of these terms
+  .not('opinion')                                   // Exclude this term
+  .build();
+
+// Source filtering
+const sourceQuery = client.query()
+  .phrase('artificial intelligence')
+  .fromDomain('techcrunch.com', true)               // Exact domain match
+  .fromCountry('US')                                // Filter by country
+  .inLanguage('english')                            // Filter by language
+  .build();
+
+// Tone analysis
+const toneQuery = client.query()
+  .phrase('economic outlook')
+  .withTone('>', 5)                                 // Articles with positive tone
+  .withAbsoluteTone('>', 7)                         // High emotional content
+  .withPositiveTone(3)                              // Alternative for positive tone
+  .withNegativeTone(-3)                             // Articles with negative tone
+  .withNeutralTone(1)                               // Articles with neutral tone
+  .withHighEmotion(5)                               // Articles with high emotional content
+  .build();
+
+// Content filtering
+const themeQuery = client.query()
+  .phrase('climate change')
+  .withTheme('ENV_CLIMATE')                         // Filter by GDELT theme
+  .build();
+
+// Image-specific queries
+const imageQuery = client.query()
+  .phrase('natural disaster')
+  .withImageTag('flood')                            // Filter by image tag
+  .withImageWebTag('emergency')                     // Filter by web image tag
+  .withImageOCR('rescue')                           // Filter by text in image
+  .withImageFaceCount('>', 3)                       // Images with more than 3 faces
+  .withImageFaceTone('<', 0)                        // Images with negative face expressions
+  .withImageWebCount('>', 10)                       // Images with high web presence
+  .withNovelImages(0.7)                             // Filter for novel images
+  .withPopularImages(0.8)                           // Filter for popular images
+  .build();
+
+// Advanced text operations
+const textQuery = client.query()
+  .withProximity(5, ['climate', 'action'])          // Terms within 5 words of each other
+  .withRepeat(3, 'urgent')                          // Term appears at least 3 times
+  .build();
+
+// Custom query components
+const customQuery = client.query()
+  .custom('domain:nytimes.com OR domain:washingtonpost.com')  // Custom query string
+  .build();
+
+// Grouping for complex logic
+const groupedQuery = client.query()
+  .phrase('climate change')
+  .group()                                          // Group the previous terms
+  .anyOf('policy', 'legislation', 'regulation')     // Match any of these terms
+  .build();
+```
+
 #### Article-specific Query Builder
 
+The specialized ArticleQueryBuilder provides methods tailored for article searches:
+
 ```typescript
 const articleQuery = client.articleQuery()
-  .breakingNews()
+  .breakingNews()                                   // Filter for breaking news
+  .opinions()                                       // Filter for opinion pieces
+  .localNews('New York')                            // Filter for local news
   .fromDomain('cnn.com')
+  .withPositiveTone(2)
   .build();
 
 const articles = await client.getArticles(articleQuery);
@@ -180,26 +257,41 @@ const articles = await client.getArticles(articleQuery);
 
 #### Image-specific Query Builder
 
+The specialized ImageQueryBuilder provides methods tailored for image searches:
+
 ```typescript
 const imageQuery = client.imageQuery()
-  .disasters()
+  .disasters()                                      // Filter for disaster images
+  .politicalEvents()                                // Filter for political events
+  .medicalContent()                                 // Filter for medical content
+  .positiveImages()                                 // Filter for positive images
+  .negativeImages()                                 // Filter for negative images
   .withNovelImages()
   .build();
 
 const images = await client.getImages(imageQuery);
 ```
 
-### Query Validation
+#### Query Validation and Optimization
 
-Validate queries before execution:
+The client provides methods to validate and optimize queries:
 
 ```typescript
+// Validate a query before execution
 const validation = client.validateQuery('climate change AND (weather OR temperature)');
 
 if (!validation.valid) {
   console.error('Query errors:', validation.errors);
 }
 
+// Check query complexity
+const complexity = client.getQueryComplexity('climate change AND (weather OR temperature)');
+console.log(`Query complexity: ${complexity}`);
+
+// Check for balanced parentheses and quotes
+const balanced = client.hasBalancedQuotes('climate "change');
+console.log(`Query has balanced quotes: ${balanced}`);
+
 // Get optimization suggestions
 const suggestions = client.getQueryOptimizations('very complex query here');
 suggestions.forEach(suggestion => {
@@ -212,6 +304,8 @@ suggestions.forEach(suggestion => {
 | Method | Description |
 |--------|-------------|
 | `getArticles()` | Search for news articles |
+| `getArticlesWithContent()` | Search for news articles and fetch their full content |
+| `fetchContentForArticles()` | Fetch full content for existing articles |
 | `getImages()` | Search for images and photos |
 | `getTimeline()` | Get timeline of coverage volume |
 | `getTimelineWithArticles()` | Get timeline with article details |
@@ -354,6 +448,132 @@ response.articles.forEach(article => {
 });
 ```
 
+### Article Content Fetching
+
+The client provides methods to fetch and parse the full content of articles returned by GDELT API calls:
+
+```typescript
+import { GdeltClient } from 'gdelt-ts-client';
+
+const client = new GdeltClient({
+  contentFetcher: {
+    concurrencyLimit: 3,        // Maximum number of concurrent requests
+    requestDelay: 1500,         // Delay between requests to the same domain (ms)
+    userAgent: 'MyApp/1.0.0',   // Custom user agent
+    respectRobotsTxt: true      // Whether to respect robots.txt
+  }
+});
+
+// Fetch articles with content in one call
+const articlesWithContent = await client.getArticlesWithContent({
+  query: 'climate change',
+  timespan: '1d',
+  maxrecords: 10
+});
+
+console.log(`Fetched ${articlesWithContent.contentStats.successCount} articles with content`);
+
+articlesWithContent.articles.forEach(article => {
+  if (article.content) {
+    console.log(`Title: ${article.title}`);
+    console.log(`Word Count: ${article.content.wordCount}`);
+    console.log(`Content Preview: ${article.content.text.substring(0, 200)}...`);
+  }
+});
+```
+
+#### Fetching Content for Existing Articles
+
+You can also fetch content for articles you've already retrieved:
+
+```typescript
+// Get articles first
+const articles = await client.getArticles({
+  query: 'technology news',
+  timespan: '2h',
+  maxrecords: 20
+});
+
+// Then selectively fetch content
+const articlesWithContent = await client.fetchContentForArticles(
+  articles.articles,
+  {
+    allowedDomains: ['bbc.com', 'reuters.com', 'apnews.com'],  // Only fetch from these domains
+    concurrencyLimit: 2,                                       // Limit concurrent requests
+    onProgress: (completed, total) => {                        // Track progress
+      console.log(`Progress: ${completed}/${total} articles processed`);
+    },
+    includeFailures: true,      // Include articles where content fetching failed
+    parseContent: true,         // Parse and clean the content
+    includeRawHTML: false       // Don't include raw HTML in the response
+  }
+);
+
+// Filter successful content fetches
+const successfulArticles = articlesWithContent.filter(
+  article => article.contentResult.success
+);
+
+console.log(`Successfully fetched content for ${successfulArticles.length} articles`);
+```
+
+#### Content Fetching Configuration
+
+The content fetcher can be configured with various options:
+
+```typescript
+const client = new GdeltClient({
+  contentFetcher: {
+    // Request control
+    concurrencyLimit: 5,        // Maximum concurrent requests
+    requestDelay: 1000,         // Delay between requests to same domain (ms)
+    timeout: 10000,             // Request timeout (ms)
+    maxRetries: 2,              // Maximum retries per request
+    
+    // Rate limiting
+    maxRequestsPerSecond: 1,    // Maximum requests per second per domain
+    maxRequestsPerMinute: 30,   // Maximum requests per minute per domain
+    
+    // Ethical scraping
+    respectRobotsTxt: true,     // Respect robots.txt files
+    userAgent: 'MyApp/1.0.0',   // User agent string
+    
+    // Domain filtering
+    skipDomains: ['paywall.com', 'subscription-only.com'],  // Skip these domains
+    
+    // Request behavior
+    followRedirects: true,      // Follow redirects
+    maxRedirects: 5,            // Maximum redirects to follow
+    customHeaders: {            // Custom headers for requests
+      'Accept-Language': 'en-US,en;q=0.9'
+    }
+  }
+});
+```
+
+#### Content Fetching Statistics
+
+You can access statistics about the content fetching process:
+
+```typescript
+const result = await client.getArticlesWithContent({
+  query: 'breaking news',
+  timespan: '1h',
+  maxrecords: 50
+});
+
+// Check content fetching statistics
+console.log('Content Fetching Statistics:');
+console.log(`Success Rate: ${(result.contentStats.successCount / result.count * 100).toFixed(1)}%`);
+console.log(`Average Fetch Time: ${result.contentStats.averageFetchTime}ms`);
+console.log(`Total Time: ${result.contentStats.totalFetchTime}ms`);
+
+// Log failure reasons
+Object.entries(result.contentStats.failureReasons).forEach(([reason, count]) => {
+  console.log(`${reason}: ${count} failures`);
+});
+```
+
 ### Timeline Analysis
 
 ```typescript

package/dist/client.d.ts

@@ -1,8 +1,11 @@
 import { ETimespanUnit } from './constants';
 import { IGdeltApiBaseParams, IGdeltClientConfig } from './interfaces/api-parameters';
-import { IArticleListResponse, IImageCollageResponse, ITimelineResponse, ITimelineBreakdownResponse, IToneChartResponse, IWordCloudResponse } from './interfaces/api-responses';
+import { IArticleListResponse, IImageCollageResponse, ITimelineResponse, ITimelineBreakdownResponse, IToneChartResponse, IWordCloudResponse, IArticle } from './interfaces/api-responses';
+import { IArticleListWithContentResponse, IArticleWithContent } from './interfaces/content-responses';
+import { IFetchContentOptions } from './interfaces/content-fetcher';
 import { TimespanUnitType, TimespanString, ComplexQuery } from './types/enhanced-types';
 import { QueryBuilder, ArticleQueryBuilder, ImageQueryBuilder } from './types/query-builder';
+import { ContentFetcherService } from './services/content-fetcher';
 /**
  * GDELT API Client
  * A strongly-typed client for interacting with the GDELT API
@@ -38,6 +41,11 @@ export declare class GdeltClient {
      * @private
      */
     private readonly _retryDelay;
+    /**
+     * The content fetcher service for article content retrieval
+     * @private
+     */
+    private readonly _contentFetcher;
     /**
      * Creates a new GDELT API client
      * @param config - The client configuration
@@ -59,6 +67,7 @@ export declare class GdeltClient {
     /**
      * Transforms API response data without mutating the original
      * @param data - The original response data
+     * @param mode - The API mode used for the request
      * @returns The transformed response data
      * @private
      */
@@ -228,5 +237,54 @@ export declare class GdeltClient {
      * Enhanced response validation and transformation
      */
     private _transformAndValidateResponse;
+    /**
+     * Fetch articles with their full content
+     * @param params - Standard GDELT API parameters
+     * @param fetchOptions - Content fetching options
+     * @returns Articles with full content included
+     */
+    getArticlesWithContent(params: IGdeltApiBaseParams, fetchOptions?: IFetchContentOptions): Promise<IArticleListWithContentResponse>;
+    /**
+     * Fetch articles with their full content (overload with query string)
+     * @param query - Query string or ComplexQuery
+     * @param fetchOptions - Content fetching options
+     * @returns Articles with full content included
+     */
+    getArticlesWithContent(query: string | ComplexQuery, fetchOptions?: IFetchContentOptions): Promise<IArticleListWithContentResponse>;
+    /**
+     * Fetch content for existing articles
+     * @param articles - Array of article objects
+     * @param options - Content fetching options
+     * @returns Articles with content added
+     */
+    fetchContentForArticles(articles: IArticle[], options?: IFetchContentOptions): Promise<IArticleWithContent[]>;
+    /**
+     * Get content fetcher service instance
+     * @returns Content fetcher service
+     */
+    getContentFetcher(): ContentFetcherService;
+    /**
+     * Merge articles with content results
+     * @param articles - Original article response
+     * @param contentResults - Content fetch results
+     * @returns Merged response with content
+     * @private
+     */
+    private _mergeArticlesWithContent;
+    /**
+     * Merge articles array with content results
+     * @param articles - Original articles array
+     * @param contentResults - Content fetch results
+     * @returns Merged articles with content
+     * @private
+     */
+    private _mergeArticlesWithContentArray;
+    /**
+     * Calculate content fetching statistics
+     * @param contentResults - Content fetch results
+     * @returns Content statistics
+     * @private
+     */
+    private _calculateContentStats;
 }
 //# sourceMappingURL=client.d.ts.map

package/dist/client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,aAAa,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAEnB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,YAAY,EAKb,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EAElB,MAAM,uBAAuB,CAAC;AAa/B;;;GAGG;AACH,qBAAa,WAAW;IACtB;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAE/C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IAEzC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IAEjC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC;;;OAGG;gBACgB,MAAM,CAAC,EAAE,kBAAkB;IAoB9C;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAI7B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAsCvB;;;;;OAKG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAsDzB;;;;;OAKG;YACW,YAAY;IAmC1B;;;;OAIG;IACU,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IACpF;;;;;OAKG;IACU,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA8B7H;;;;OAIG;IACU,SAAS,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IACnF;;;;;OAKG;IACU,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA8B5H;;;;OAIG;IACU,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACjF;;;;;OAKG;IACU,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,iBAAiB,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqCrJ;;;;OAIG;IACU,uBAAuB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAC7F;;;;;OAKG;IACU,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6BtI;;;;OAIG;IACU,qBAAqB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAQpG;;;;OAIG;IACU,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAQnG;;;;OAIG;IACU,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrF;;;;;OAKG;IACU,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6B9H;;;;OAIG;IACU,YAAY,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IACnF;;;;;OAKG;IACU,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAsC5H;;;;OAIG;IACU,gBAAgB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQvF;;;;OAIG;IACU,mBAAmB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQ1F;;;;;OAKG;IACI,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,MAAM;IACjE;;;;;OAKG;IACI,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,cAAc;IAkB5E;;OAEG;IACI,KAAK,IAAI,YAAY;IAI5B;;OAEG;IACI,YAAY,IAAI,mBAAmB;IAI1C;;OAEG;IACI,UAAU,IAAI,iBAAiB;IAItC;;OAEG;IACI,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE;IA0BzE;;OAEG;IACI,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAyBrD;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+B/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuE7B;;OAEG;IACH,OAAO,CAAC,6BAA6B;CAUtC"}
+{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AACA,OAAO,EAIL,aAAa,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAEnB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,iBAAiB,EACjB,0BAA0B,EAC1B,kBAAkB,EAClB,kBAAkB,EAClB,QAAQ,EACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,+BAA+B,EAC/B,mBAAmB,EAEpB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACpE,OAAO,EAEL,gBAAgB,EAChB,cAAc,EACd,YAAY,EAKb,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,YAAY,EACZ,mBAAmB,EACnB,iBAAiB,EAElB,MAAM,uBAAuB,CAAC;AAY/B,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AAEnE;;;GAGG;AACH,qBAAa,WAAW;IACtB;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAgB;IAE/C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAElC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IAEzC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAU;IAEjC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IAErC;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAwB;IAExD;;;OAGG;gBACgB,MAAM,CAAC,EAAE,kBAAkB;IAuB9C;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAI7B;;;;OAIG;IACH,OAAO,CAAC,eAAe;IAsCvB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;IAkE1B;;;;;OAKG;IACH,OAAO,CAAC,iBAAiB;IAsDzB;;;;;OAKG;YACW,YAAY;IAmC1B;;;;OAIG;IACU,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,oBAAoB,CAAC;IACpF;;;;;OAKG;IACU,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA8B7H;;;;OAIG;IACU,SAAS,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,qBAAqB,CAAC;IACnF;;;;;OAKG;IACU,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA8B5H;;;;OAIG;IACU,WAAW,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACjF;;;;;OAKG;IACU,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,iBAAiB,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAqCrJ;;;;OAIG;IACU,uBAAuB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAC7F;;;;;OAKG;IACU,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6BtI;;;;OAIG;IACU,qBAAqB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAQpG;;;;OAIG;IACU,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,0BAA0B,CAAC;IAQnG;;;;OAIG;IACU,eAAe,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IACrF;;;;;OAKG;IACU,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6B9H;;;;OAIG;IACU,YAAY,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IACnF;;;;;OAKG;IACU,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,YAAY,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAsC5H;;;;OAIG;IACU,gBAAgB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQvF;;;;OAIG;IACU,mBAAmB,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAQ1F;;;;;OAKG;IACI,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,GAAG,MAAM;IACjE;;;;;OAKG;IACI,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,gBAAgB,GAAG,cAAc;IAkB5E;;OAEG;IACI,KAAK,IAAI,YAAY;IAI5B;;OAEG;IACI,YAAY,IAAI,mBAAmB;IAI1C;;OAEG;IACI,UAAU,IAAI,iBAAiB;IAItC;;OAEG;IACI,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,EAAE,MAAM,EAAE,CAAA;KAAE;IA0BzE;;OAEG;IACI,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE;IAyBrD;;OAEG;IACH,OAAO,CAAC,uBAAuB;IA+B/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAuE7B;;OAEG;IACH,OAAO,CAAC,6BAA6B;IAarC;;;;;OAKG;IACU,sBAAsB,CACjC,MAAM,EAAE,mBAAmB,EAC3B,YAAY,CAAC,EAAE,oBAAoB,GAClC,OAAO,CAAC,+BAA+B,CAAC;IAC3C;;;;;OAKG;IACU,sBAAsB,CACjC,KAAK,EAAE,MAAM,GAAG,YAAY,EAC5B,YAAY,CAAC,EAAE,oBAAoB,GAClC,OAAO,CAAC,+BAA+B,CAAC;IAgC3C;;;;;OAKG;IACU,uBAAuB,CAClC,QAAQ,EAAE,QAAQ,EAAE,EACpB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAcjC;;;OAGG;IACI,iBAAiB,IAAI,qBAAqB;IAIjD;;;;;;OAMG;IACH,OAAO,CAAC,yBAAyB;IA2CjC;;;;;;OAMG;IACH,OAAO,CAAC,8BAA8B;IAyCtC;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;CA2C/B"}

package/dist/client.js

@@ -19,6 +19,7 @@ const enhanced_types_1 = require("./types/enhanced-types");
 const query_builder_1 = require("./types/query-builder");
 const type_guards_1 = require("./types/type-guards");
 const lookups_1 = require("./types/lookups");
+const content_fetcher_1 = require("./services/content-fetcher");
 /**
  * GDELT API Client
  * A strongly-typed client for interacting with the GDELT API
@@ -46,6 +47,8 @@ class GdeltClient {
                 'Content-Type': 'application/json'
             }
         });
+        // Initialize content fetcher service
+        this._contentFetcher = new content_fetcher_1.ContentFetcherService(config === null || config === void 0 ? void 0 : config.contentFetcher);
     }
     /**
      * Creates a timespan string from a timespan object
@@ -97,11 +100,18 @@ class GdeltClient {
     /**
      * Transforms API response data without mutating the original
      * @param data - The original response data
+     * @param mode - The API mode used for the request
      * @returns The transformed response data
      * @private
      */
-    _transformResponse(data) {
-        if (!data || typeof data !== 'object') {
+    _transformResponse(data, mode) {
+        // For backward compatibility with tests, still throw error for null/undefined
+        if (!data) {
+            throw new Error('Invalid response data: expected object');
+        }
+        // For empty responses that are objects, handle gracefully
+        // For non-objects, throw error for backward compatibility
+        if (typeof data !== 'object') {
             throw new Error('Invalid response data: expected object');
         }
         // Create a new object without mutating the original
@@ -110,6 +120,28 @@ class GdeltClient {
         if (!('status' in transformedData)) {
             transformedData['status'] = 'ok';
         }
+        // Ensure expected array properties exist based on the mode
+        if (mode === constants_1.EMode.articleList && !('articles' in transformedData)) {
+            transformedData['articles'] = [];
+        }
+        if (mode === constants_1.EMode.imageCollageInfo && !('images' in transformedData)) {
+            transformedData['images'] = [];
+        }
+        if ((mode === constants_1.EMode.timelineVolume || mode === constants_1.EMode.timelineVolumeInfo || mode === constants_1.EMode.timelineTone)
+            && !('timeline' in transformedData)) {
+            transformedData['timeline'] = [];
+        }
+        if ((mode === constants_1.EMode.timelineLanguage || mode === constants_1.EMode.timelineSourceCountry)
+            && !('data' in transformedData)) {
+            transformedData['data'] = [];
+        }
+        if (mode === constants_1.EMode.toneChart && !('tonechart' in transformedData)) {
+            transformedData['tonechart'] = [];
+        }
+        if ((mode === constants_1.EMode.wordCloudImageTags || mode === constants_1.EMode.wordCloudImageWebTags)
+            && !('wordcloud' in transformedData)) {
+            transformedData['wordcloud'] = [];
+        }
         // Add count property for article list responses
         if ('articles' in transformedData && Array.isArray(transformedData["articles"]) && !('count' in transformedData)) {
             transformedData['count'] = transformedData["articles"].length;
@@ -205,7 +237,7 @@ class GdeltClient {
                 throw new Error(response.data);
             }
             // Transform response data without mutating original
-            const transformedData = this._transformResponse(response.data);
+            const transformedData = this._transformResponse(response.data, params.mode);
             return transformedData;
         });
     }
@@ -342,7 +374,7 @@ class GdeltClient {
             const response = yield this._makeRequest(Object.assign(Object.assign({}, finalParams), { mode: constants_1.EMode.toneChart, format: constants_1.EFormat.json }));
             // Enhanced validation with type guard
             const validatedResponse = this._transformAndValidateResponse(response, type_guards_1.TypeGuards.isToneChartResponse);
-            // Ensure the response has the expected structure
+            // Ensure the response has the expected structure with tonechart property
             if (!validatedResponse.tonechart) {
                 throw new Error('Invalid response format from GDELT API: missing tonechart property');
             }
@@ -552,6 +584,158 @@ class GdeltClient {
         }
         return transformedData;
     }
+    getArticlesWithContent(paramsOrQuery, fetchOptions) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // 1. Get articles from GDELT API
+            let articles;
+            if (typeof paramsOrQuery === 'string') {
+                // Call the string overload
+                articles = yield this.getArticles(paramsOrQuery);
+            }
+            else if ('mode' in paramsOrQuery) {
+                // Call the IGdeltApiBaseParams overload
+                articles = yield this.getArticles(paramsOrQuery);
+            }
+            else {
+                // Call the ComplexQuery overload (which is handled by the string overload)
+                articles = yield this.getArticles(paramsOrQuery);
+            }
+            // 2. Extract URLs from articles
+            const urls = articles.articles.map(article => article.url);
+            // 3. Fetch content for all URLs
+            const contentResults = yield this._contentFetcher.fetchMultipleArticleContent(urls, fetchOptions);
+            // 4. Merge content with article metadata
+            return this._mergeArticlesWithContent(articles, contentResults);
+        });
+    }
+    /**
+     * Fetch content for existing articles
+     * @param articles - Array of article objects
+     * @param options - Content fetching options
+     * @returns Articles with content added
+     */
+    fetchContentForArticles(articles, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // Extract URLs from articles
+            const urls = articles.map(article => article.url);
+            // Fetch content for all URLs
+            const contentResults = yield this._contentFetcher.fetchMultipleArticleContent(urls, options);
+            // Merge content with article metadata
+            return this._mergeArticlesWithContentArray(articles, contentResults);
+        });
+    }
+    /**
+     * Get content fetcher service instance
+     * @returns Content fetcher service
+     */
+    getContentFetcher() {
+        return this._contentFetcher;
+    }
+    /**
+     * Merge articles with content results
+     * @param articles - Original article response
+     * @param contentResults - Content fetch results
+     * @returns Merged response with content
+     * @private
+     */
+    _mergeArticlesWithContent(articles, contentResults) {
+        var _a;
+        const articlesWithContent = [];
+        const resultMap = new Map();
+        // Create map of URL to content result
+        for (const result of contentResults) {
+            resultMap.set(result.url, result);
+        }
+        // Merge articles with content
+        for (const article of articles.articles) {
+            const contentResult = resultMap.get(article.url);
+            if (contentResult) {
+                const articleWithContent = Object.assign(Object.assign({}, article), { content: contentResult.success ? (_a = contentResult.content) !== null && _a !== void 0 ? _a : null : null });
+                if (!contentResult.success && contentResult.error) {
+                    articleWithContent.contentError = contentResult.error;
+                }
+                if (contentResult.timing) {
+                    articleWithContent.contentTiming = contentResult.timing;
+                }
+                articlesWithContent.push(articleWithContent);
+            }
+        }
+        // Calculate statistics
+        const contentStats = this._calculateContentStats(contentResults);
+        return Object.assign(Object.assign({}, articles), { articles: articlesWithContent, contentStats });
+    }
+    /**
+     * Merge articles array with content results
+     * @param articles - Original articles array
+     * @param contentResults - Content fetch results
+     * @returns Merged articles with content
+     * @private
+     */
+    _mergeArticlesWithContentArray(articles, contentResults) {
+        var _a;
+        // If there are no articles or no content results, return empty array
+        if (!articles.length || !(contentResults === null || contentResults === void 0 ? void 0 : contentResults.length)) {
+            return [];
+        }
+        const articlesWithContent = [];
+        const resultMap = new Map();
+        // Create map of URL to content result
+        for (const result of contentResults) {
+            resultMap.set(result.url, result);
+        }
+        // Merge articles with content
+        for (const article of articles) {
+            const contentResult = resultMap.get(article.url);
+            if (contentResult) {
+                const articleWithContent = Object.assign(Object.assign({}, article), { content: contentResult.success ? ((_a = contentResult.content) !== null && _a !== void 0 ? _a : null) : null });
+                if (!contentResult.success && contentResult.error) {
+                    articleWithContent.contentError = contentResult.error;
+                }
+                if (contentResult.timing) {
+                    articleWithContent.contentTiming = contentResult.timing;
+                }
+                articlesWithContent.push(articleWithContent);
+            }
+        }
+        return articlesWithContent;
+    }
+    /**
+     * Calculate content fetching statistics
+     * @param contentResults - Content fetch results
+     * @returns Content statistics
+     * @private
+     */
+    _calculateContentStats(contentResults) {
+        var _a;
+        const totalArticles = contentResults.length;
+        const successfulFetches = contentResults.filter(r => r.success).length;
+        const failedFetches = contentResults.filter(r => !r.success).length;
+        const totalFetchTime = Math.max(...contentResults.map(r => r.timing.totalTime));
+        const averageFetchTime = contentResults.length > 0 ?
+            contentResults.reduce((sum, r) => sum + r.timing.fetchTime, 0) / contentResults.length : 0;
+        const averageParseTime = contentResults.length > 0 ?
+            contentResults.reduce((sum, r) => sum + r.timing.parseTime, 0) / contentResults.length : 0;
+        const totalWords = contentResults
+            .filter(r => r.success && r.content)
+            .reduce((sum, r) => { var _a, _b; return sum + ((_b = (_a = r.content) === null || _a === void 0 ? void 0 : _a.wordCount) !== null && _b !== void 0 ? _b : 0); }, 0);
+        const failureReasons = {};
+        for (const result of contentResults) {
+            if (!result.success && result.error) {
+                const reason = result.error.code || 'UNKNOWN';
+                failureReasons[reason] = ((_a = failureReasons[reason]) !== null && _a !== void 0 ? _a : 0) + 1;
+            }
+        }
+        return {
+            totalArticles,
+            successfulFetches,
+            failedFetches,
+            averageFetchTime,
+            averageParseTime,
+            totalFetchTime,
+            totalWords,
+            failureReasons
+        };
+    }
 }
 exports.GdeltClient = GdeltClient;
 //# sourceMappingURL=client.js.map