http-request-manager 18.12.3 → 18.12.5

package/fesm2022/http-request-manager.mjs

@@ -883,28 +883,66 @@ class StreamingProcessor {
         this.buffer = '';
         this.contentType = '';
         this.maxBufferSize = 10 * 1024 * 1024; // 10MB limit
+        this.lastParsedLength = 0; // Track parsed position to avoid duplicates
         this.streamConfig = config || { streamType: StreamType.AI_STREAMING };
     }
     /**
-     * Process HTTP events and extract streaming data
+     * Process HTTP events and extract streaming data with progress
      */
     process(event) {
         switch (event.type) {
             case HttpEventType.ResponseHeader:
                 this.contentType = event.headers?.get('content-type') || '';
+                // Read total from response header if configured
+                if (this.streamConfig.totalHeader && event.headers) {
+                    const totalVal = event.headers.get(this.streamConfig.totalHeader);
+                    if (totalVal !== undefined && totalVal !== null) {
+                        const parsed = parseInt(totalVal, 10);
+                        if (!isNaN(parsed)) {
+                            this.totalFromHeader = parsed;
+                        }
+                    }
+                }
                 return null;
             case HttpEventType.DownloadProgress:
                 if (event.partialText) {
                     this.appendToBuffer(event.partialText);
                     const parsedData = this.parseBuffer();
-                    return parsedData.length > 0 ? [parsedData[0]] : null;
+                    // Only return NEW items since last parse (progressive updates)
+                    const newItems = parsedData.slice(this.lastParsedLength);
+                    this.lastParsedLength = parsedData.length;
+                    // Calculate progress
+                    const progress = {
+                        received: this.lastParsedLength,
+                        total: this.totalFromHeader,
+                        percent: this.totalFromHeader
+                            ? Math.round((this.lastParsedLength / this.totalFromHeader) * 100)
+                            : 0,
+                        stage: 'streaming'
+                    };
+                    return {
+                        data: newItems.length > 0 ? newItems : [],
+                        progress
+                    };
                 }
                 return null;
             case HttpEventType.Response:
                 if (event.body) {
                     this.appendToBuffer(event.body);
                     const parsedData = this.parseBuffer();
-                    return parsedData.length > 0 ? parsedData : null;
+                    // Calculate final progress
+                    const progress = {
+                        received: parsedData.length,
+                        total: this.totalFromHeader,
+                        percent: this.totalFromHeader
+                            ? Math.round((parsedData.length / this.totalFromHeader) * 100)
+                            : 100,
+                        stage: 'complete'
+                    };
+                    return {
+                        data: parsedData.length > 0 ? parsedData : [],
+                        progress
+                    };
                 }
                 return null;
             default:
@@ -943,6 +981,8 @@ class StreamingProcessor {
     reset() {
         this.buffer = '';
         this.contentType = '';
+        this.lastParsedLength = 0;
+        this.totalFromHeader = undefined;
     }
     /**
      * Update configuration
@@ -2793,19 +2833,6 @@ class UploadValidationErrorModel {
     }
 }
 
-class UserData {
-    constructor(ldap = '', name = '', email = '', color = RandomPaletteColor()) {
-        this.ldap = ldap;
-        this.name = name;
-        this.email = email;
-        this.color = color;
-    }
-    static adapt(item) {
-        const userName = `${item?.first_name} ${item?.last_name}`;
-        return new UserData(item?.ldap, userName, item?.email, item?.color);
-    }
-}
-
 class RequestService extends WebsocketService {
     constructor() {
         super(...arguments);
@@ -2816,6 +2843,13 @@ class RequestService extends WebsocketService {
         this.isPending$ = this.isPending.asObservable();
         this.progress = new BehaviorSubject(0);
         this.progress$ = this.progress.asObservable();
+        this.streamProgress = new BehaviorSubject({
+            received: 0,
+            total: undefined,
+            percent: 0,
+            stage: 'connecting'
+        });
+        this.streamProgress$ = this.streamProgress.asObservable();
     }
     // Implementation
     getRecordRequest(options) {
@@ -2884,7 +2918,12 @@ class RequestService extends WebsocketService {
     }
     requestStreaming(options) {
         return (source$) => {
-            return source$.pipe(map(data => {
+            return source$.pipe(tap(output => {
+                // Update progress from stream output
+                this.progress.next(output.progress.received);
+                this.streamProgress.next(output.progress);
+            }), map(output => {
+                const data = output.data;
                 if (!data || (Array.isArray(data) && data.length === 0)) {
                     return data;
                 }
@@ -3586,6 +3625,7 @@ class HTTPManagerService extends RequestService {
         this.data$ = this.data.asObservable();
         this.polling$ = new Subject();
         this.config = ApiRequest.adapt();
+        this.streamProgress$ = this.streamProgress.asObservable();
         this.config = (configOptions) ? ApiRequest.adapt(configOptions.httpRequestOptions) : this.config;
     }
     // ═══════════════════════════════════════════════════════════════════════════
@@ -3722,6 +3762,12 @@ class HTTPManagerService extends RequestService {
     getRequest(options, params) {
         this.isPending.next(true);
         this.data.next(null);
+        this.streamProgress.next({
+            received: 0,
+            total: undefined,
+            percent: 0,
+            stage: 'connecting'
+        });
         const updatedOptions = this.defineReqOptions(options, params);
         const func = this.getRecordRequest;
         const requests = this.createRequest(func, updatedOptions);
@@ -3730,9 +3776,19 @@ class HTTPManagerService extends RequestService {
             this.data.next(data);
             if (updatedOptions.displaySuccess)
                 this.handleSuccessWithSnackBar(updatedOptions.successMessage);
-        })).pipe(finalize(() => this.isPending.next(false)), catchError((err) => {
+        }), finalize(() => {
+            this.streamProgress.next({
+                ...this.streamProgress.value,
+                stage: 'complete'
+            });
+            this.isPending.next(false);
+        }), catchError((err) => {
             if (updatedOptions.displayError)
                 this.handleErrorWithSnackBar(err, updatedOptions.errorMessage || err?.message);
+            this.streamProgress.next({
+                ...this.streamProgress.value,
+                stage: 'error'
+            });
             this.isPending.next(false);
             return this.handleError(err);
         }));
@@ -4154,7 +4210,8 @@ class RequestSignalsService extends WebsocketService {
     }
     requestStreamingOperator(options) {
         return (source$) => {
-            return source$.pipe(map(data => {
+            return source$.pipe(map(output => {
+                const data = output.data;
                 if (!data || (Array.isArray(data) && data.length === 0)) {
                     return data;
                 }
@@ -4891,12 +4948,13 @@ class ApiRequest {
 }
 
 class RequestOptions {
-    constructor(path = [], headers = {}) {
+    constructor(path = [], headers = {}, forceRefresh) {
         this.path = path;
         this.headers = headers;
+        this.forceRefresh = forceRefresh;
     }
     static adapt(item) {
-        return new RequestOptions(item?.path, item?.headers);
+        return new RequestOptions(item?.path, item?.headers, item?.forceRefresh);
     }
 }
 
@@ -5772,9 +5830,12 @@ class DbService extends Dexie {
         await this.dbReady;
         const safeTableName = this.cleanTableName(tableName);
         const safeSchema = schema.trim();
-        if (this.tableExists(safeTableName))
-            return;
         const currentSchema = this.getCurrentSchema();
+        const existingSchema = currentSchema[safeTableName]?.trim();
+        // No-op only when table already exists and schema is identical.
+        if (this.tableExists(safeTableName) && existingSchema === safeSchema) {
+            return;
+        }
         console.log('Current Schema before update:', currentSchema);
         currentSchema[safeTableName] = safeSchema;
         const nextVersion = this.verno + 1;
@@ -5788,6 +5849,7 @@ class DbService extends Dexie {
             const created = this.tables.some(t => t.name === safeTableName);
             if (!created) {
                 console.error(`CRITICAL: Table ${safeTableName} was NOT created after upgrade! Tables found:`, this.tables.map(t => t.name));
+                throw new Error(`Table '${safeTableName}' was not created after schema upgrade`);
             }
             else {
                 console.log(`Database opened successfully version ${this.verno}. Table ${safeTableName} verified.`);
@@ -5795,15 +5857,24 @@ class DbService extends Dexie {
         }
         catch (err) {
             console.error('Error opening database after schema update:', err);
+            throw err;
         }
     }
     async DBOpened() {
+        try {
+            await this.dbReady;
+        }
+        catch (err) {
+            console.error('DBOpened: init failed', err);
+            return false;
+        }
         if (!this.isOpen()) {
             try {
                 await this.open();
                 return true;
             }
             catch (err) {
+                console.error('DBOpened: open failed', err);
                 return false;
             }
         }
@@ -5898,7 +5969,11 @@ class DatabaseManagerService extends DbService {
         }));
     }
     createDatabaseTable(tableDef) {
-        return from(this.createTable(tableDef.table, tableDef.schema)).pipe(switchMap(() => this.DBOpened()));
+        const tableName = this.cleanTableName(tableDef.table);
+        return from(this.createTable(tableDef.table, tableDef.schema)).pipe(switchMap(() => from(this.DBOpened())), map((opened) => opened && this.tableExists(tableName)), catchError((error) => {
+            console.error(`createDatabaseTable: failed for table '${tableName}'`, error);
+            return of(false);
+        }));
     }
     updateDatabaseTableSchema(tableDef) {
         return from(this.createTable(tableDef.table, tableDef.schema)).pipe(switchMap(() => this.getDatabaseTable(tableDef.table)));
@@ -5945,26 +6020,23 @@ class DatabaseManagerService extends DbService {
     }
     createTableRecords(table, records) {
         const tableName = this.cleanTableName(table);
-        return from(this.DBOpened()).pipe(switchMap(() => {
+        return from(this.DBOpened()).pipe(switchMap((opened) => {
+            if (!opened) {
+                console.error(`createTableRecords: DB not open. Cannot write to '${tableName}'.`);
+                return EMPTY;
+            }
             if (!this.tables.some(t => t.name === tableName)) {
                 console.error(`createTableRecords: Table '${tableName}' does not exist in DB version ${this.verno}. Available:`, this.tables.map(t => t.name));
                 return EMPTY;
             }
             const tableInstance = this.table(tableName);
-            // Extract schema fields directly from the table instance
-            const schema = tableInstance.schema;
-            const validFields = [
-                schema.primKey.name,
-                ...schema.indexes.map(idx => idx.name)
-            ];
+            // Keep full object payload; Dexie stores non-indexed properties as well.
             const insertRecords = records.map((record) => {
-                const objectFromSchema = {};
-                validFields.forEach((field) => {
-                    if (field) {
-                        objectFromSchema[field] = record[field] ?? null;
-                    }
-                });
-                return objectFromSchema;
+                const payload = { ...(record || {}) };
+                if (payload.id === undefined || payload.id === null || payload.id === '') {
+                    delete payload.id;
+                }
+                return payload;
             });
             console.log(`createTableRecords: Bulk putting ${insertRecords.length} records into ${tableName}`);
             return from(tableInstance.bulkPut(insertRecords)).pipe(map(() => insertRecords));
@@ -5978,17 +6050,14 @@ class DatabaseManagerService extends DbService {
     updateTableRecords(table, records) {
         const tableName = this.cleanTableName(table);
         return this.getDatabaseTable(tableName).pipe(switchMap((tableData) => {
-            return this.getDatabaseTableSchema(tableName).pipe(switchMap((schema) => {
-                const insertRecords = records.map((record) => {
-                    const objectFromSchema = {};
-                    schema.forEach((field) => {
-                        const data = record[field] ?? null;
-                        objectFromSchema[field] = data;
-                    });
-                    return objectFromSchema;
-                });
-                return from(tableData.bulkPut(insertRecords)).pipe(map(() => insertRecords));
-            }));
+            const insertRecords = records.map((record) => {
+                const payload = { ...(record || {}) };
+                if (payload.id === undefined || payload.id === null || payload.id === '') {
+                    delete payload.id;
+                }
+                return payload;
+            });
+            return from(tableData.bulkPut(insertRecords)).pipe(map(() => insertRecords));
         }));
     }
     deleteTableRecord(table, id) {
@@ -6106,6 +6175,15 @@ class HTTPManagerStateService extends ComponentStore {
         this.hasDatabase = false;
         this.streamedResponse = [];
         this.shouldRetry = true;
+        this.volatileHeaders = new Set([
+            'authorization',
+            'x-request-id',
+            'x-correlation-id',
+            'x-trace-id',
+            'x-amzn-trace-id',
+            'date',
+            'if-none-match'
+        ]);
         this.wsRetryAttempts = new BehaviorSubject(0);
         this.wsRetryAttempts$ = this.wsRetryAttempts.asObservable();
         this.messages = new BehaviorSubject([]);
@@ -6375,22 +6453,25 @@ class HTTPManagerStateService extends ComponentStore {
             }
         }))))));
         this.initDBStorage = this.effect((trigger$) => trigger$.pipe(tap(() => {
+            console.log('[initDBStorage effect] Triggered, checking conditions:', {
+                dataType: this.dataType,
+                isARRAY: this.dataType === DataType.ARRAY,
+                hasAdapter: !!this.apiOptions?.adapter,
+                hasTable: !!this.databaseOptions?.table,
+                tableValue: this.databaseOptions?.table
+            });
             if (this.dataType !== DataType.ARRAY)
                 console.warn('Database storage requires dataType to be ARRAY');
             if (!this.apiOptions.adapter)
-                console.warn('Database storage requires an adapter to define the data shape');
+                console.warn('Database storage adapter missing, using minimal or inferred schema');
             if (this.databaseOptions && this.databaseOptions?.table === '')
                 console.warn('Database storage requires a table name');
-        }), filter(() => this.dataType === DataType.ARRAY && !!this.apiOptions.adapter && !!this.databaseOptions?.table), switchMap(() => {
-            const sampleData = this.apiOptions.adapter?.({}) || {};
-            const schemaKeys = Object.keys(sampleData).filter(key => sampleData[key] !== undefined);
-            let schema = '++id';
-            if (schemaKeys.length > 0) {
-                const otherKeys = schemaKeys.filter(k => k !== 'id');
-                if (otherKeys.length > 0) {
-                    schema += ', ' + otherKeys.join(', ');
-                }
-            }
+        }), filter(() => {
+            const shouldProceed = this.dataType === DataType.ARRAY && !!this.databaseOptions?.table;
+            console.log('[initDBStorage effect] Filter result:', shouldProceed);
+            return shouldProceed;
+        }), switchMap(() => {
+            const schema = this.buildSchemaFromAdapter();
             const tableDef = TableSchemaDef.adapt({
                 table: this.databaseOptions?.table,
                 schema: schema
@@ -6479,6 +6560,7 @@ class HTTPManagerStateService extends ComponentStore {
             }
         }), concatMap(() => {
             if (this.hasDatabase && this.databaseOptions?.table) {
+                this.clearRequestCacheMetadata(this.databaseOptions.table);
                 const currentData = this.get()?.data;
                 const idsToDelete = Array.isArray(currentData) ? currentData.map((r) => r.id) : [];
                 if (idsToDelete.length > 0) {
@@ -6493,35 +6575,86 @@ class HTTPManagerStateService extends ComponentStore {
         this.fetchRecords = (options) => this.effect(() => of(RequestOptions.adapt(options)).pipe(switchMap(() => {
             this.streamedResponse = [];
             const requestOptions = this.updateRequestOptions(options?.headers);
+            const effectiveParams = this.getEffectiveParams(options?.path);
+            const requestSignature = this.buildRequestSignature('GET', requestOptions, effectiveParams);
             const fetchFromAPI = () => {
-                return this.httpManagerService.getRequest(requestOptions, options?.path).pipe(tap((data) => {
-                    data = (!data) ? (this.dataType === DataType.ARRAY) ? [] : {} : data;
+                return this.httpManagerService.getRequest(requestOptions, effectiveParams).pipe(tap((data) => {
+                    // Extract array from paginated response if needed
+                    const arrayData = (data?.results && Array.isArray(data.results)) ? data.results : data;
+                    data = (!arrayData) ? (this.dataType === DataType.ARRAY) ? [] : {} : arrayData;
                     this.setData$(data);
                 }), concatMap((data) => {
-                    if (this.hasDatabase && this.databaseOptions?.table && Array.isArray(data) && data.length > 0) {
+                    // Extract array from paginated response for database storage
+                    const dbData = (data?.results && Array.isArray(data.results)) ? data.results : data;
+                    if (this.hasDatabase && this.databaseOptions?.table && Array.isArray(dbData) && dbData.length > 0) {
+                        const tableName = this.databaseOptions.table;
                         this.localStorageManagerService.updateStore({
-                            name: this.databaseOptions.table,
+                            name: tableName,
                             data: { ...this.databaseOptions, ...{ expires: this.utils.expires(this.databaseOptions.expiresIn) } }
                         });
-                        return this.dbManagerService.createTableRecords(this.databaseOptions.table, data);
+                        const schema = this.buildSchemaFromSample(dbData[0]);
+                        const tableDef = TableSchemaDef.adapt({ table: tableName, schema });
+                        const schemaSignature = this.buildSchemaSignature(schema);
+                        // Always ensure table exists immediately before writing to avoid stale schema/store races.
+                        return this.localStorageManagerService.store$(tableName).pipe(take(1), switchMap((storeData) => {
+                            const storedSchemaSignature = this.getStoredSchemaSignature(storeData);
+                            const schemaChanged = !!storedSchemaSignature && storedSchemaSignature !== schemaSignature;
+                            const ensureTable$ = schemaChanged
+                                ? this.dbManagerService.clearTable(tableName).pipe(switchMap(() => this.dbManagerService.createDatabaseTable(tableDef)))
+                                : this.dbManagerService.createDatabaseTable(tableDef);
+                            return ensureTable$.pipe(switchMap((created) => {
+                                if (!created) {
+                                    console.warn('[DB STORAGE] Table create/open not ready, skipping DB write for this payload:', { table: tableName });
+                                    return of(data);
+                                }
+                                return this.dbManagerService.createTableRecords(tableName, dbData).pipe(tap(() => this.saveRequestCacheMetadata(tableName, 'GET', requestSignature, schemaSignature, options)));
+                            }));
+                        }), catchError((error) => {
+                            console.error('[DB STORAGE] Failed to ensure table and write records:', { table: tableName, schema, error });
+                            return of(data);
+                        }));
                     }
                     return of(data);
                 }));
             };
+            console.log('[DB STORAGE] Checking database storage:', {
+                hasDatabase: this.hasDatabase,
+                table: this.databaseOptions?.table,
+                databaseOptions: this.databaseOptions
+            });
             if (this.hasDatabase && this.databaseOptions?.table) {
                 return this.dbManagerService.databaseExists().pipe(switchMap((dbExists) => {
                     if (!dbExists) {
                         const initObs = this.initDBStorageAsync();
-                        return initObs.pipe(switchMap(() => fetchFromAPI()));
+                        return initObs.pipe(switchMap(() => fetchFromAPI()), catchError((error) => {
+                            console.warn('[DB STORAGE] initDBStorageAsync failed when DB did not exist, continuing with API:', error);
+                            return fetchFromAPI();
+                        }));
                     }
                     return this.dbManagerService.hasDatabaseTable(this.databaseOptions.table).pipe(switchMap((tableExists) => {
                         if (!tableExists) {
                             const initObs = this.initDBStorageAsync();
-                            return initObs.pipe(switchMap(() => fetchFromAPI()));
+                            return initObs.pipe(switchMap(() => fetchFromAPI()), catchError((error) => {
+                                console.warn('[DB STORAGE] initDBStorageAsync failed when table missing, continuing with API:', error);
+                                return fetchFromAPI();
+                            }));
                         }
                         return this.localStorageManagerService.store$(this.databaseOptions.table).pipe(take(1), switchMap((storeData) => {
+                            const cached = this.getRequestCacheMetadata(storeData, 'GET');
+                            const sameSignature = !!cached?.signature && cached.signature === requestSignature;
+                            const forceRefresh = !!options?.forceRefresh;
+                            const storedSchemaSignature = this.getStoredSchemaSignature(storeData);
                             const expires = storeData?.expires || 0;
                             const hasExpired = expires > 0 && this.utils.hasExpired(expires);
+                            if (!forceRefresh && sameSignature && !hasExpired) {
+                                return this.dbManagerService.getTableRecords(this.databaseOptions.table).pipe(switchMap((dbData) => {
+                                    if (Array.isArray(dbData) && dbData.length > 0) {
+                                        this.setData$(dbData);
+                                        return of(dbData);
+                                    }
+                                    return fetchFromAPI();
+                                }));
+                            }
                             if (hasExpired) {
                                 return this.dbManagerService.clearTable(this.databaseOptions.table).pipe(switchMap(() => fetchFromAPI()), tap(() => {
                                     this.localStorageManagerService.updateStore({
@@ -6530,12 +6663,27 @@ class HTTPManagerStateService extends ComponentStore {
                                     });
                                 }));
                             }
+                            const expectedSchema = this.buildSchemaFromAdapter();
+                            const expectedSchemaSignature = this.buildSchemaSignature(expectedSchema);
+                            if (storedSchemaSignature && storedSchemaSignature !== expectedSchemaSignature) {
+                                const tableDef = TableSchemaDef.adapt({ table: this.databaseOptions.table, schema: expectedSchema });
+                                return this.dbManagerService.clearTable(this.databaseOptions.table).pipe(switchMap(() => this.dbManagerService.createDatabaseTable(tableDef)), switchMap(() => fetchFromAPI()));
+                            }
                             return this.dbManagerService.getTableRecords(this.databaseOptions.table).pipe(switchMap((dbData) => {
                                 if (Array.isArray(dbData) && dbData.length > 0) {
                                     this.setData$(dbData);
                                     return of(dbData);
                                 }
                                 return fetchFromAPI();
+                            }), catchError((error) => {
+                                const tableName = this.databaseOptions.table;
+                                console.warn('[DB STORAGE] getTableRecords failed, recreating table and falling back to API:', { table: tableName, error });
+                                const schema = this.buildSchemaFromAdapter();
+                                const tableDef = TableSchemaDef.adapt({ table: tableName, schema });
+                                return this.dbManagerService.createDatabaseTable(tableDef).pipe(switchMap(() => fetchFromAPI()), catchError((recreateError) => {
+                                    console.error('[DB STORAGE] Failed to recreate table after read error, continuing with API only:', recreateError);
+                                    return fetchFromAPI();
+                                }));
                             }));
                         }));
                     }));
@@ -6659,7 +6807,7 @@ class HTTPManagerStateService extends ComponentStore {
                 if (res.length > 0)
                     this.setData$(res);
                 this.streamedResponse = res;
-            }), scan((acc, res) => {
+            }), concatMap((res) => this.persistStreamDataToDb(res, options)), scan((acc, res) => {
                 const previous = acc.current;
                 const current = res;
                 return { previous, current };
@@ -6679,8 +6827,52 @@ class HTTPManagerStateService extends ComponentStore {
         }), switchMap((options) => {
             const requestOptions = this.updateRequestOptions(options?.headers);
             requestOptions.stream = true;
+            const effectiveParams = this.getEffectiveParams(options?.path);
+            const requestSignature = this.buildRequestSignature('STREAM', requestOptions, effectiveParams);
             console.log('[DEBUG] Making streaming request:', requestOptions);
-            return this.httpManagerService.getRequest(requestOptions, options?.path)
+            if (this.hasDatabase && this.databaseOptions?.table && !options?.forceRefresh) {
+                return this.localStorageManagerService.store$(this.databaseOptions.table).pipe(take(1), switchMap((storeData) => {
+                    const cached = this.getRequestCacheMetadata(storeData, 'STREAM');
+                    const sameSignature = !!cached?.signature && cached.signature === requestSignature;
+                    const expires = storeData?.expires || 0;
+                    const hasExpired = expires > 0 && this.utils.hasExpired(expires);
+                    if (sameSignature && !hasExpired) {
+                        return this.dbManagerService.getTableRecords(this.databaseOptions.table).pipe(switchMap((dbData) => {
+                            if (Array.isArray(dbData) && dbData.length > 0) {
+                                return of({ data: dbData, fromCache: true });
+                            }
+                            return this.httpManagerService.getRequest(requestOptions, effectiveParams).pipe(map((apiData) => ({ data: apiData, fromCache: false })));
+                        }));
+                    }
+                    return this.httpManagerService.getRequest(requestOptions, effectiveParams).pipe(map((apiData) => ({ data: apiData, fromCache: false })));
+                })).pipe(tap((packet) => {
+                    const res = packet?.data;
+                    console.log('[DEBUG] Streaming response received:', res);
+                    if (res && res.length > 0) {
+                        console.log('[DEBUG] Updating state with streaming data:', res);
+                        this.setData$(res);
+                        this.streamedResponse = res;
+                    }
+                    else {
+                        console.log('[DEBUG] No streaming data or empty array:', res);
+                    }
+                    // Reset pending once we have a response packet (cache or network)
+                    this.httpManagerService.isPending.next(false);
+                }), concatMap((packet) => {
+                    if (packet?.fromCache) {
+                        return of(packet?.data);
+                    }
+                    return this.persistStreamDataToDb(packet?.data, options);
+                }), map((res) => {
+                    console.log('[DEBUG] Returning data to subscribers:', res);
+                    return res;
+                }), catchError((error) => {
+                    console.error('[DEBUG] Streaming error:', error);
+                    this.httpManagerService.isPending.next(false);
+                    return of([]);
+                }));
+            }
+            return this.httpManagerService.getRequest(requestOptions, effectiveParams)
                 .pipe(tap((res) => {
                 console.log('[DEBUG] Streaming response received:', res);
                 // Always update state with streaming data
@@ -6692,7 +6884,9 @@ class HTTPManagerStateService extends ComponentStore {
                 else {
                     console.log('[DEBUG] No streaming data or empty array:', res);
                 }
-            }), map((res) => {
+                // Reset pending once we have a response packet
+                this.httpManagerService.isPending.next(false);
+            }), concatMap((res) => this.persistStreamDataToDb(res, options)), map((res) => {
                 console.log('[DEBUG] Returning data to subscribers:', res);
                 return res; // Return the data so subscribers can receive it
             }), catchError((error) => {
@@ -6719,7 +6913,11 @@ class HTTPManagerStateService extends ComponentStore {
                         encrypted: false,
                     })
                 });
-                this.initDBStorage();
+                // Use initDBStorageAsync directly - the effect initDBStorage requires subscription
+                this.initDBStorageAsync().subscribe({
+                    next: () => console.log('[Constructor] Database storage initialized'),
+                    error: (err) => console.error('[Constructor] Database storage initialization failed:', err)
+                });
             }
         }
         catch (error) {
@@ -6773,8 +6971,24 @@ class HTTPManagerStateService extends ComponentStore {
         this.dataType = (dataType) ? dataType : DataType.ARRAY;
         // Only update database options if a database parameter is explicitly provided
         if (database !== undefined) {
+            console.log('[setApiRequestOptions] Database config:', {
+                database,
+                hasTable: !!database?.table,
+                tableValue: database?.table
+            });
             this.hasDatabase = (database?.table) ? true : false;
-            this.databaseOptions = (this.hasDatabase) ? DatabaseStorage.adapt(database) : undefined;
+            const adapted = DatabaseStorage.adapt(database);
+            console.log('[setApiRequestOptions] DatabaseStorage.adapt result:', adapted);
+            this.databaseOptions = (this.hasDatabase) ? adapted : undefined;
+            // Trigger database table creation if table is configured
+            if (this.hasDatabase && this.databaseOptions?.table) {
+                console.log('[setApiRequestOptions] Initializing database storage for table:', this.databaseOptions.table);
+                // Use initDBStorageAsync directly instead of the effect - effects need subscription to run
+                this.initDBStorageAsync().subscribe({
+                    next: () => console.log('[setApiRequestOptions] Database storage initialized successfully'),
+                    error: (err) => console.error('[setApiRequestOptions] Database storage initialization failed:', err)
+                });
+            }
         }
         if (this.apiOptions.ws && this.apiOptions.ws.id !== '') {
             // Auto-prefix channel ID for private state manager channels
@@ -6896,11 +7110,20 @@ class HTTPManagerStateService extends ComponentStore {
         this.setData$(data);
     }
     updateArrayState(currentData, newData) {
+        // For non-streaming requests (GET), REPLACE data entirely
+        // For streaming requests, MERGE with existing data incrementally
+        if (this.streamedResponse.length === 0) {
+            // GET request: return new data as-is (replace)
+            return newData;
+        }
+        // Streaming: merge with existing data
         const filterCurrentData = () => {
             const ids = this.streamedResponse.map((obj) => obj.id);
             return currentData.filter(obj => (obj.id) ? ids.includes(obj.id) : obj);
         };
-        const filteredCurrentData = (this.httpManagerService.isPending.value) ? currentData : filterCurrentData();
+        const filteredCurrentData = (this.httpManagerService.isPending.value)
+            ? currentData
+            : filterCurrentData();
         const updatedData = filteredCurrentData.map(item => {
             const newItem = newData.find(newItem => {
                 const hasId = (newItem?.id && item?.id) ? true : false;
@@ -6917,32 +7140,92 @@ class HTTPManagerStateService extends ComponentStore {
         return [...updatedData, ...addedData];
     }
     initDBStorageAsync() {
+        console.log('[initDBStorageAsync] Starting initialization:', {
+            dataType: this.dataType,
+            hasAdapter: !!this.apiOptions?.adapter,
+            table: this.databaseOptions?.table,
+            databaseOptions: this.databaseOptions
+        });
         if (this.dataType !== DataType.ARRAY) {
             console.warn('Database storage requires dataType to be ARRAY');
             return of(null);
         }
         if (!this.apiOptions.adapter) {
-            console.warn('Database storage requires an adapter to define the data shape');
-            return of(null);
+            console.warn('Database storage adapter missing, using minimal or inferred schema');
         }
         if (!this.databaseOptions?.table) {
             console.warn('Database storage requires a table name');
             return of(null);
         }
+        const schema = this.buildSchemaFromAdapter();
+        const tableDef = TableSchemaDef.adapt({
+            table: this.databaseOptions?.table,
+            schema: schema
+        });
+        return this.dbManagerService.createDatabaseTable(tableDef).pipe(tap((created) => {
+            if (created && this.databaseOptions?.table) {
+                this.saveSchemaSignature(this.databaseOptions.table, this.buildSchemaSignature(schema));
+            }
+        }));
+    }
+    buildSchemaFromAdapter() {
         const sampleData = this.apiOptions.adapter?.({}) || {};
-        const schemaKeys = Object.keys(sampleData).filter(key => sampleData[key] !== undefined);
+        return this.buildSchemaFromSample(sampleData);
+    }
+    buildSchemaFromSample(sampleData) {
+        const schemaKeys = Object.keys(sampleData || {}).filter(key => sampleData[key] !== undefined);
         let schema = '++id';
         if (schemaKeys.length > 0) {
-            const otherKeys = schemaKeys.filter(k => k !== 'id');
+            const otherKeys = schemaKeys.filter((k) => k !== 'id');
             if (otherKeys.length > 0) {
                 schema += ', ' + otherKeys.join(', ');
             }
         }
-        const tableDef = TableSchemaDef.adapt({
-            table: this.databaseOptions?.table,
-            schema: schema
-        });
-        return this.dbManagerService.createDatabaseTable(tableDef);
+        return schema;
+    }
+    persistStreamDataToDb(payload, streamOptions) {
+        if (!this.hasDatabase || !this.databaseOptions?.table) {
+            return of(payload);
+        }
+        const dbData = (payload?.results && Array.isArray(payload.results))
+            ? payload.results
+            : Array.isArray(payload)
+                ? payload
+                : payload
+                    ? [payload]
+                    : [];
+        if (dbData.length === 0) {
+            return of(payload);
+        }
+        const tableName = this.databaseOptions.table;
+        const requestOptions = this.updateRequestOptions(streamOptions?.headers);
+        requestOptions.stream = true;
+        const effectiveParams = this.getEffectiveParams(streamOptions?.path);
+        const requestSignature = this.buildRequestSignature('STREAM', requestOptions, effectiveParams);
+        this.localStorageManagerService.updateStore({
+            name: tableName,
+            data: { ...this.databaseOptions, ...{ expires: this.utils.expires(this.databaseOptions.expiresIn) } }
+        });
+        const schema = this.buildSchemaFromSample(dbData[0]);
+        const schemaSignature = this.buildSchemaSignature(schema);
+        const tableDef = TableSchemaDef.adapt({ table: tableName, schema });
+        return this.localStorageManagerService.store$(tableName).pipe(take(1), switchMap((storeData) => {
+            const storedSchemaSignature = this.getStoredSchemaSignature(storeData);
+            const schemaChanged = !!storedSchemaSignature && storedSchemaSignature !== schemaSignature;
+            const ensureTable$ = schemaChanged
+                ? this.dbManagerService.clearTable(tableName).pipe(switchMap(() => this.dbManagerService.createDatabaseTable(tableDef)))
+                : this.dbManagerService.createDatabaseTable(tableDef);
+            return ensureTable$.pipe(switchMap((created) => {
+                if (!created) {
+                    console.warn('[DB STORAGE] Stream table create/open not ready, skipping DB write for this chunk:', { table: tableName });
+                    return of(payload);
+                }
+                return this.dbManagerService.createTableRecords(tableName, dbData).pipe(tap(() => this.saveRequestCacheMetadata(tableName, 'STREAM', requestSignature, schemaSignature, streamOptions)), map(() => payload));
+            }));
+        }), map(() => payload), catchError((error) => {
+            console.error('[DB STORAGE] Failed to persist streaming payload:', { table: tableName, schema, error });
+            return of(payload);
+        }));
     }
     // WEBSOCKET COMMUNICATION (STATE MANAGER)
     wsCommunication(method, path) {
@@ -7238,6 +7521,7 @@ class HTTPManagerStateService extends ComponentStore {
         const tableName = this.databaseOptions.table;
         this.dbManagerService.clearTable(tableName).subscribe({
             next: () => {
+                this.clearRequestCacheMetadata(tableName);
                 if (this.dataType === DataType.ARRAY) {
                     this.setData$([]);
                 }
@@ -7260,6 +7544,115 @@ class HTTPManagerStateService extends ComponentStore {
             : { ...options.headers };
         return options;
     }
+    normalizeObject(value) {
+        if (Array.isArray(value)) {
+            return value.map((item) => this.normalizeObject(item));
+        }
+        if (value && typeof value === 'object') {
+            return Object.keys(value)
+                .sort()
+                .reduce((acc, key) => {
+                acc[key] = this.normalizeObject(value[key]);
+                return acc;
+            }, {});
+        }
+        return value;
+    }
+    filterHeaders(headers) {
+        const source = headers || {};
+        return Object.keys(source).reduce((acc, key) => {
+            if (!this.volatileHeaders.has(key.toLowerCase())) {
+                acc[key] = source[key];
+            }
+            return acc;
+        }, {});
+    }
+    resolvePath(params) {
+        const basePath = Array.isArray(this.apiOptions.path) ? this.apiOptions.path : [];
+        const effective = this.getEffectiveParams(params);
+        return effective ? [...basePath, ...effective] : [...basePath];
+    }
+    getEffectiveParams(params) {
+        if (!Array.isArray(params) || params.length === 0) {
+            return undefined;
+        }
+        const basePath = Array.isArray(this.apiOptions.path) ? this.apiOptions.path : [];
+        if (basePath.length !== params.length) {
+            return params;
+        }
+        const normalizePart = (value) => {
+            if (value && typeof value === 'object') {
+                return JSON.stringify(this.normalizeObject(value));
+            }
+            return String(value);
+        };
+        const samePath = params.every((part, index) => normalizePart(part) === normalizePart(basePath[index]));
+        return samePath ? undefined : params;
+    }
+    buildRequestSignature(method, requestOptions, params) {
+        const signaturePayload = {
+            method,
+            server: requestOptions.server,
+            path: this.resolvePath(params),
+            headers: this.filterHeaders(requestOptions.headers),
+            stream: !!requestOptions.stream,
+            streamType: requestOptions.streamType || null
+        };
+        return JSON.stringify(this.normalizeObject(signaturePayload));
+    }
+    buildSchemaSignature(schema) {
+        return JSON.stringify(this.normalizeObject(schema.split(',').map((part) => part.trim()).filter(Boolean)));
+    }
+    getRequestCacheMetadata(storeData, type) {
+        return storeData?.requestCache?.[type] || null;
+    }
+    getStoredSchemaSignature(storeData) {
+        return storeData?.schemaSignature || null;
+    }
+    saveSchemaSignature(tableName, schemaSignature) {
+        this.localStorageManagerService.store$(tableName).pipe(take(1), tap((storeData) => {
+            this.localStorageManagerService.updateStore({
+                name: tableName,
+                data: {
+                    ...(storeData || {}),
+                    schemaSignature,
+                }
+            });
+        })).subscribe();
+    }
+    saveRequestCacheMetadata(tableName, type, signature, schemaSignature, options) {
+        this.localStorageManagerService.store$(tableName).pipe(take(1), tap((storeData) => {
+            const currentCache = storeData?.requestCache || {};
+            this.localStorageManagerService.updateStore({
+                name: tableName,
+                data: {
+                    ...(storeData || {}),
+                    schemaSignature: schemaSignature || storeData?.schemaSignature || null,
+                    requestCache: {
+                        ...currentCache,
+                        [type]: {
+                            signature,
+                            savedAt: Date.now(),
+                            path: this.resolvePath(options?.path),
+                            headers: this.filterHeaders(options?.headers)
+                        }
+                    }
+                }
+            });
+        })).subscribe();
+    }
+    clearRequestCacheMetadata(tableName) {
+        this.localStorageManagerService.store$(tableName).pipe(take(1), tap((storeData) => {
+            if (!storeData)
+                return;
+            const updated = { ...(storeData || {}) };
+            delete updated.requestCache;
+            this.localStorageManagerService.updateStore({
+                name: tableName,
+                data: updated
+            });
+        })).subscribe();
+    }
     static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "18.2.14", ngImport: i0, type: HTTPManagerStateService, deps: [{ token: API_OPTS }, { token: "dataType" }, { token: DatabaseStorage }], target: i0.ɵɵFactoryTarget.Injectable }); }
     static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "18.2.14", ngImport: i0, type: HTTPManagerStateService }); }
 }
@@ -8596,13 +8989,13 @@ class StateManagerDemoService extends HTTPManagerStateService {
         const sampleOptions = RequestOptions.adapt({ path: [data.id] });
         this.deleteRecord(sampleOptions);
     }
-    streamRequest() {
+    streamRequest(options) {
         console.log('[DEMO SERVICE] streamRequest called');
         const headers = {
             auth: "sample-auth-token"
         };
         console.log('[DEMO SERVICE] Calling fetchStream with headers:', headers);
-        this.fetchStream();
+        this.fetchStream(options);
     }
     static { this.ɵfac = i0.ɵɵngDeclareFactory({ minVersion: "12.0.0", version: "18.2.14", ngImport: i0, type: StateManagerDemoService, deps: [], target: i0.ɵɵFactoryTarget.Injectable }); }
     static { this.ɵprov = i0.ɵɵngDeclareInjectable({ minVersion: "12.0.0", version: "18.2.14", ngImport: i0, type: StateManagerDemoService }); }
@@ -8849,11 +9242,23 @@ class RequestManagerStateDemoComponent {
         return { apiOptions: apiOptions, path: pathReq };
     }
     onSetStateOptions() {
-        if (!this.isValid)
+        const dbValue = this.database;
+        console.log('[onSetStateOptions] Called, checking form values:', {
+            isValid: this.isValid,
+            database: dbValue,
+            hasTable: !!dbValue?.table,
+            tableValue: dbValue?.table,
+            dataType: this.dataType
+        });
+        if (!this.isValid) {
+            console.log('[onSetStateOptions] Form invalid, aborting');
             return;
+        }
         const reqParams = this.compileRequest();
-        const db = DatabaseStorage.adapt(this.database);
+        const db = DatabaseStorage.adapt(dbValue);
+        console.log('[onSetStateOptions] DatabaseStorage.adapt result:', db);
         const type = this.dataType === "ARRAY" ? DataType.ARRAY : DataType.OBJECT;
+        console.log('[onSetStateOptions] Calling setAPIOptions with:', { db, type, hasTable: !!db?.table });
         this.stateManagerDemoService.setAPIOptions(reqParams.apiOptions, type, db);
         this.requestForm.markAsPristine();
     }
@@ -8888,8 +9293,13 @@ class RequestManagerStateDemoComponent {
         reqParams.apiOptions.stream = true;
         reqParams.apiOptions.streamType = this.streamType;
         this.requestType = 'STREAM';
+        const streamOptions = RequestOptions.adapt({
+            path: reqParams.path,
+            headers: reqParams.apiOptions.headers,
+            forceRefresh: false,
+        });
         console.log('[COMPONENT] Calling streamRequest...');
-        this.stateManagerDemoService.streamRequest();
+        this.stateManagerDemoService.streamRequest(streamOptions);
     }
     errorHandling(err, type) {
         console.log(err, type);
@@ -9934,6 +10344,19 @@ i0.ɵɵngDeclareClassMetadata({ minVersion: "12.0.0", version: "18.2.14", ngImpo
                 }]
         }] });
 
+class UserData {
+    constructor(ldap = '', name = '', email = '', color = RandomPaletteColor()) {
+        this.ldap = ldap;
+        this.name = name;
+        this.email = email;
+        this.color = color;
+    }
+    static adapt(item) {
+        const userName = `${item?.first_name} ${item?.last_name}`;
+        return new UserData(item?.ldap, userName, item?.email, item?.color);
+    }
+}
+
 class StateDataRequestService extends HTTPManagerStateService {
     constructor() {
         super(ApiRequest.adapt({