irdata_js 0.1.0 → 0.2.0

package/README.md

@@ -54,18 +54,52 @@ if (code) {
 
 ### 3. Fetch Data
 
-Once authenticated, you can call any endpoint using `getData`. This method handles authentication headers and automatically follows S3 links if returned by the API.
+Once authenticated, you can call any endpoint using `getData`. This method handles authentication headers, automatically follows S3 links if returned by the API, and provides metadata about the response.
 
 ```javascript
 try {
   // Call an endpoint directly
-  const memberInfo = await client.getData('/member/info');
-  console.log(memberInfo);
+  const { data, metadata } = await client.getData('/member/info');
+  
+  console.log(data); // The actual API response
+  console.log(metadata.sizeBytes); // Response size in bytes
+  console.log(metadata.chunksDetected); // Boolean indicating if data is chunked
 } catch (error) {
   console.error('Failed to fetch member info:', error);
 }
 ```
 
+### 4. Handling Large Datasets (Chunks)
+
+Some iRacing endpoints (like large result sets) return data in multiple "chunks" hosted on S3. When `metadata.chunksDetected` is true, you can use the library to fetch the rest of the data.
+
+#### Fetch all chunks at once
+
+```javascript
+const result = await client.getData('/results/get');
+
+if (result.metadata.chunksDetected) {
+  // Fetch and merge all chunks into a single array
+  const { data: allResults } = await client.getChunks(result.data);
+  console.log('Total results:', allResults.length);
+}
+```
+
+#### Fetch chunks individually (Pagination)
+
+For extremely large datasets, you might want to fetch chunks one by one:
+
+```javascript
+if (result.metadata.chunksDetected) {
+  const totalChunks = result.data.chunk_info.chunk_file_names.length;
+  
+  for (let i = 0; i < totalChunks; i++) {
+    const { data: chunk } = await client.getChunk(result.data, i);
+    console.log(`Processing chunk ${i + 1}/${totalChunks}`);
+  }
+}
+```
+
 ## Development
 
 ### Build
@@ -103,4 +137,4 @@ This repository includes a local development proxy server to test the OAuth flow
 
 ## License
 
-ISC
+MIT

package/dist/client.d.ts

@@ -9,20 +9,71 @@ interface ClientConfig {
         tokenEndpoint?: string;
     };
 }
+export interface ChunkInfo {
+    chunk_size: number;
+    num_chunks: number;
+    rows: number;
+    base_download_url: string;
+    chunk_file_names: string[];
+}
+export interface DataResult<T> {
+    data: T;
+    metadata: {
+        s3LinkFollowed: boolean;
+        chunksDetected: boolean;
+        sizeBytes: number;
+    };
+}
+export interface ChunkResult<T> {
+    data: T;
+    metadata: {
+        sizeBytes: number;
+    };
+}
 export declare class IRacingClient {
     auth: AuthManager;
     private apiUrl;
     private fileProxyUrl?;
     constructor(config?: ClientConfig);
+    private calculateSize;
+    /**
+     * Internal method to perform request and return data + size.
+     */
+    private requestInternal;
     /**
      * Performs a fetch request with authentication headers.
      * Does NOT automatically follow "link" responses.
      */
     request<T>(endpoint: string, options?: RequestInit): Promise<T>;
     private handleErrorResponse;
+    /**
+     * Helper to fetch data from an external URL (e.g. S3), handling the file proxy if configured.
+     */
+    private fetchExternal;
     /**
      * Fetches data from an endpoint, automatically following any S3 links returned.
+     * Returns metadata about the operation.
+     */
+    getData<T>(endpoint: string): Promise<DataResult<T>>;
+    private hasChunks;
+    /**
+     * Fetches a specific chunk from a response containing chunk_info.
+     *
+     * @param data The response object containing chunk_info
+     * @param chunkIndex The index of the chunk to fetch (0-based)
+     * @returns The content of the chunk and metadata
+     */
+    getChunk<T>(data: unknown, chunkIndex: number): Promise<ChunkResult<T[]>>;
+    /**
+     * Fetches multiple chunks and merges the results into a single array.
+     *
+     * @param data The response object containing chunk_info
+     * @param options Options for fetching chunks (start index, limit count)
+     * @returns A merged array of data from the requested chunks and total size
      */
-    getData<T>(endpoint: string): Promise<T>;
+    getChunks<T>(data: unknown, options?: {
+        start?: number;
+        limit?: number;
+    }): Promise<ChunkResult<T[]>>;
 }
 export {};

package/dist/client.js

@@ -9,12 +9,22 @@ export class IRacingClient {
         this.fileProxyUrl = config.fileProxyUrl;
         this.auth = new AuthManager(config.auth);
     }
+    calculateSize(response, data) {
+        const cl = response.headers.get('content-length');
+        if (cl)
+            return parseInt(cl, 10);
+        try {
+            return new TextEncoder().encode(JSON.stringify(data)).length;
+        }
+        catch {
+            return 0;
+        }
+    }
     /**
-     * Performs a fetch request with authentication headers.
-     * Does NOT automatically follow "link" responses.
+     * Internal method to perform request and return data + size.
      */
-    async request(endpoint, options = {}) {
-        // Remove leading slash from endpoint if present to avoid double slashes if apiUrl has trailing slash
+    async requestInternal(endpoint, options = {}) {
+        // Remove leading slash from endpoint if present
         const cleanEndpoint = endpoint.startsWith('/') ? endpoint.substring(1) : endpoint;
         const cleanApiUrl = this.apiUrl.endsWith('/') ? this.apiUrl : `${this.apiUrl}/`;
         const url = `${cleanApiUrl}${cleanEndpoint}`;
@@ -27,7 +37,7 @@ export class IRacingClient {
                 'Content-Type': 'application/json',
             },
         };
-        const response = await fetch(url, mergedOptions);
+        let response = await fetch(url, mergedOptions);
         if (!response.ok) {
             if (response.status === 401) {
                 // Try to refresh token
@@ -44,22 +54,28 @@ export class IRacingClient {
                                 'Content-Type': 'application/json',
                             },
                         };
-                        const retryResponse = await fetch(url, retryOptions);
-                        if (retryResponse.ok) {
-                            return retryResponse.json();
-                        }
-                        // If retry fails, use the retryResponse for error handling below
-                        return this.handleErrorResponse(retryResponse);
+                        response = await fetch(url, retryOptions);
                     }
                 }
                 catch (refreshError) {
                     console.error('Token refresh failed during request retry:', refreshError);
-                    // Fall through to original 401 error handling
                 }
             }
-            return this.handleErrorResponse(response);
+            if (!response.ok) {
+                return this.handleErrorResponse(response);
+            }
         }
-        return response.json();
+        const data = await response.json();
+        const sizeBytes = this.calculateSize(response, data);
+        return { data, sizeBytes };
+    }
+    /**
+     * Performs a fetch request with authentication headers.
+     * Does NOT automatically follow "link" responses.
+     */
+    async request(endpoint, options = {}) {
+        const { data } = await this.requestInternal(endpoint, options);
+        return data;
     }
     async handleErrorResponse(response) {
         let body;
@@ -77,33 +93,121 @@ export class IRacingClient {
         }
         throw new IRacingAPIError(`API Request failed: ${response.status} ${response.statusText}`, response.status, response.statusText, body);
     }
+    /**
+     * Helper to fetch data from an external URL (e.g. S3), handling the file proxy if configured.
+     */
+    async fetchExternal(url) {
+        let fetchUrl = url;
+        if (this.fileProxyUrl) {
+            // If a file proxy is configured, use it
+            const separator = this.fileProxyUrl.includes('?') ? '&' : '?';
+            fetchUrl = `${this.fileProxyUrl}${separator}url=${encodeURIComponent(url)}`;
+        }
+        const response = await fetch(fetchUrl);
+        if (!response.ok) {
+            return this.handleErrorResponse(response);
+        }
+        const data = await response.json();
+        const sizeBytes = this.calculateSize(response, data);
+        return { data, sizeBytes };
+    }
     /**
      * Fetches data from an endpoint, automatically following any S3 links returned.
+     * Returns metadata about the operation.
      */
     async getData(endpoint) {
-        const data = await this.request(endpoint);
+        const { data: initialData, sizeBytes: initialSize } = await this.requestInternal(endpoint);
         // Check if the response contains a generic link to S3 and follow it
-        if (data &&
-            typeof data === 'object' &&
-            'link' in data &&
-            typeof data.link === 'string') {
-            const s3Link = data.link;
+        if (initialData &&
+            typeof initialData === 'object' &&
+            'link' in initialData &&
+            typeof initialData.link === 'string') {
+            const s3Link = initialData.link;
             if (s3Link.startsWith('http')) {
-                // Fetch the S3 link without original auth headers
-                let fetchUrl = s3Link;
-                if (this.fileProxyUrl) {
-                    // If a file proxy is configured, use it
-                    // e.g. http://localhost:80/passthrough?url=...
-                    const separator = this.fileProxyUrl.includes('?') ? '&' : '?';
-                    fetchUrl = `${this.fileProxyUrl}${separator}url=${encodeURIComponent(s3Link)}`;
-                }
-                const linkResponse = await fetch(fetchUrl);
-                if (!linkResponse.ok) {
-                    return this.handleErrorResponse(linkResponse);
-                }
-                return linkResponse.json();
+                const { data: externalData, sizeBytes: externalSize } = await this.fetchExternal(s3Link);
+                return {
+                    data: externalData,
+                    metadata: {
+                        s3LinkFollowed: true,
+                        chunksDetected: this.hasChunks(externalData),
+                        sizeBytes: externalSize,
+                    },
+                };
             }
         }
-        return data;
+        return {
+            data: initialData,
+            metadata: {
+                s3LinkFollowed: false,
+                chunksDetected: this.hasChunks(initialData),
+                sizeBytes: initialSize,
+            },
+        };
+    }
+    hasChunks(data) {
+        return !!(data && typeof data === 'object' && 'chunk_info' in data);
+    }
+    /**
+     * Fetches a specific chunk from a response containing chunk_info.
+     *
+     * @param data The response object containing chunk_info
+     * @param chunkIndex The index of the chunk to fetch (0-based)
+     * @returns The content of the chunk and metadata
+     */
+    async getChunk(data, chunkIndex) {
+        const dataWithChunks = data;
+        if (!dataWithChunks || !dataWithChunks.chunk_info) {
+            throw new Error('Response does not contain chunk_info');
+        }
+        const chunkInfo = dataWithChunks.chunk_info;
+        const { base_download_url, chunk_file_names } = chunkInfo;
+        if (!base_download_url ||
+            !Array.isArray(chunk_file_names) ||
+            chunkIndex < 0 ||
+            chunkIndex >= chunk_file_names.length) {
+            throw new Error(`Invalid chunk index: ${chunkIndex} (Total chunks: ${chunk_file_names?.length || 0})`);
+        }
+        const chunkUrl = `${base_download_url}${chunk_file_names[chunkIndex]}`;
+        const { data: chunkData, sizeBytes } = await this.fetchExternal(chunkUrl);
+        return {
+            data: chunkData,
+            metadata: {
+                sizeBytes,
+            },
+        };
+    }
+    /**
+     * Fetches multiple chunks and merges the results into a single array.
+     *
+     * @param data The response object containing chunk_info
+     * @param options Options for fetching chunks (start index, limit count)
+     * @returns A merged array of data from the requested chunks and total size
+     */
+    async getChunks(data, options = {}) {
+        const dataWithChunks = data;
+        if (!dataWithChunks || !dataWithChunks.chunk_info) {
+            throw new Error('Response does not contain chunk_info');
+        }
+        const chunkInfo = dataWithChunks.chunk_info;
+        const totalChunks = chunkInfo.chunk_file_names.length;
+        const start = options.start || 0;
+        const limit = options.limit || totalChunks - start;
+        const end = Math.min(start + limit, totalChunks);
+        if (start < 0 || start >= totalChunks) {
+            throw new Error(`Invalid start index: ${start} (Total chunks: ${totalChunks})`);
+        }
+        const chunkPromises = [];
+        for (let i = start; i < end; i++) {
+            chunkPromises.push(this.getChunk(data, i));
+        }
+        const results = await Promise.all(chunkPromises);
+        const mergedData = results.flatMap((r) => r.data);
+        const totalSize = results.reduce((sum, r) => sum + r.metadata.sizeBytes, 0);
+        return {
+            data: mergedData,
+            metadata: {
+                sizeBytes: totalSize,
+            },
+        };
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "irdata_js",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "JavaScript library to interact with the iRacing /data API",
   "type": "module",
   "main": "dist/index.js",
@@ -20,10 +20,7 @@
     "format": "prettier --write .",
     "prepublishOnly": "npm test && npm run build"
   },
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/popmonkey/irdata_js.git"
-  },
+  "repository": "popmonkey/irdata_js",
   "keywords": [
     "iracing",
     "api",
@@ -31,7 +28,11 @@
     "sdk"
   ],
   "author": "",
-  "license": "ISC",
+  "license": "MIT",
+  "publishConfig": {
+    "registry": "https://registry.npmjs.org/",
+    "access": "public"
+  },
   "bugs": {
     "url": "https://github.com/popmonkey/irdata_js/issues"
   },