bulltrackers-module 1.0.755 → 1.0.757

package/functions/computation-system-v2/framework/storage/StorageManager.js

@@ -9,7 +9,7 @@
  * * FIX: Switched to bigquery.createJob for GCS imports to prevent local file path interpretation errors.
  * * FIX: Improved error logging to catch swallowed BigQuery insert errors.
  * * FIX: finalizeResults now checks for file existence to prevent "Not found" errors on empty results.
- * * FIX: Added SAFE.PARSE_JSON to MERGE statement to handle JSON type mismatch.
+ * * FIX: Added SAFE.PARSE_JSON to MERGE statement for BOTH result_data and dependency_result_hashes.
  */
 
 const { Firestore } = require('@google-cloud/firestore');
@@ -499,6 +499,8 @@ class StorageManager {
         const dataset = this.bigquery.dataset(this.config.bigquery.dataset);
         const table = dataset.table(tableName);
         
+        // Note: result_data and dependency_result_hashes are loaded as STRING from the JSON file
+        // They will be parsed into JSON during the merge step.
         const schema = [
             { name: 'date', type: 'DATE', mode: 'REQUIRED' },
             { name: 'computation_name', type: 'STRING', mode: 'REQUIRED' },
@@ -521,10 +523,6 @@ class StorageManager {
         
         await this._ensureBigQueryTable(targetTable);
 
-        // FIX: Added SAFE.PARSE_JSON() to dependency_result_hashes
-        // The source (temp table) has this as a STRING (from the JSON file).
-        // The destination (target table) has this as JSON.
-        // We must explicitly parse the string to JSON during the merge.
         const mergeQuery = `
             MERGE INTO ${fullTarget} T
             USING (
@@ -540,16 +538,33 @@ class StorageManager {
                     result_hash = S.result_hash, 
                     dependency_result_hashes = SAFE.PARSE_JSON(S.dependency_result_hashes), 
                     entity_count = S.entity_count, 
-                    result_data = S.result_data, 
+                    result_data = SAFE.PARSE_JSON(S.result_data), 
                     updated_at = S.updated_at
             WHEN NOT MATCHED THEN
                 INSERT (date, computation_name, category, entity_id, code_hash, result_hash, 
                         dependency_result_hashes, entity_count, result_data, updated_at)
                 VALUES (S.date, S.computation_name, S.category, S.entity_id, S.code_hash, S.result_hash, 
-                        SAFE.PARSE_JSON(S.dependency_result_hashes), S.entity_count, S.result_data, S.updated_at)
+                        SAFE.PARSE_JSON(S.dependency_result_hashes), S.entity_count, SAFE.PARSE_JSON(S.result_data), S.updated_at)
         `;
         
-        await this.bigquery.query({ query: mergeQuery, location: this.config.bigquery.location });
+        // UPDATE: Use createQueryJob to capture DML statistics
+        try {
+            const [job] = await this.bigquery.createQueryJob({ 
+                query: mergeQuery, 
+                location: this.config.bigquery.location 
+            });
+            
+            await job.getQueryResults(); // Wait for completion
+            
+            const metadata = await job.getMetadata();
+            const stats = metadata[0]?.statistics?.query;
+            const affectedRows = stats?.numDmlAffectedRows;
+
+            this._log('INFO', `Merge complete on ${targetTable}. Rows affected (Inserted/Updated): ${affectedRows}`);
+        } catch (e) {
+            this._logError(`Merge Failed on ${targetTable}`, e);
+            throw e;
+        }
     }
 
     async _cleanupGCSFiles(bucketName, prefix) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bulltrackers-module",
-  "version": "1.0.755",
+  "version": "1.0.757",
   "description": "Helper Functions for Bulltrackers.",
   "main": "index.js",
   "files": [