@soulcraft/brainy 5.10.0 → 5.10.1

package/CHANGELOG.md

@@ -2,6 +2,43 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [5.10.1](https://github.com/soulcraftlabs/brainy/compare/v5.10.0...v5.10.1) (2025-11-14)
+
+### 🚨 CRITICAL BUG FIX - Blob Integrity Regression
+
+**v5.10.0 regressed the v5.7.2 blob integrity bug, causing 100% VFS file read failure. This hotfix restores functionality with defense-in-depth architecture.**
+
+### Bug Description
+v5.10.0 reintroduced a critical bug where `BlobStorage.read()` was hashing wrapped binary data instead of unwrapped content, causing all blob integrity checks to fail:
+- **Symptom**: `Blob integrity check failed: <hash>` errors on every VFS file read
+- **Root Cause**: Missing defense-in-depth unwrap verification in `BlobStorage.read()`
+- **Impact**: 100% failure rate for VFS file operations in Workshop application
+
+### The Fix (v5.10.1)
+1. **Defense-in-Depth Unwrapping**: Added unwrap verification in `BlobStorage.read()` before hash check
+2. **DRY Architecture**: Created `binaryDataCodec.ts` as single source of truth for wrap/unwrap logic
+3. **Metadata Unwrapping**: Fixed metadata parsing to handle wrapped format
+4. **Comprehensive Tests**: Added 3 regression tests using `TestWrappingAdapter`
+
+### Changes
+- **NEW**: `src/storage/cow/binaryDataCodec.ts` - Single source of truth for binary data encoding/decoding
+- **FIXED**: `src/storage/cow/BlobStorage.ts` - Unwraps data and metadata before verification (lines 314, 342)
+- **REFACTORED**: `src/storage/baseStorage.ts` - Uses shared binaryDataCodec utilities (lines 332, 340)
+- **ADDED**: `tests/helpers/TestWrappingAdapter.ts` - Real wrapping adapter for testing
+- **ADDED**: 3 regression tests in `tests/unit/storage/cow/BlobStorage.test.ts`
+
+### Architecture Improvements
+- ✅ **Defense-in-Depth**: Unwrap at BOTH adapter layer (v5.7.5) and blob layer (v5.10.1)
+- ✅ **DRY Principle**: All wrap/unwrap operations use shared `binaryDataCodec.ts`
+- ✅ **Works Across ALL 8 Storage Adapters**: FileSystem, Memory, S3, GCS, Azure, R2, OPFS, Historical
+- ✅ **Prevents Future Regressions**: Real wrapping tests catch this bug class
+
+### Related Issues
+- v5.7.2: Original blob integrity bug - hashed wrapper instead of content
+- v5.7.5: First fix - added unwrap to COW adapter (necessary but insufficient)
+- v5.10.0: Regression - missing defense-in-depth in BlobStorage layer
+- v5.10.1: Complete fix - defense-in-depth + DRY architecture + comprehensive tests
+
 ### [5.9.0](https://github.com/soulcraftlabs/brainy/compare/v5.8.0...v5.9.0) (2025-11-14)
 
 - fix: resolve VFS tree corruption from blob errors (v5.8.0) (93d2d70)

package/dist/storage/baseStorage.js

@@ -10,6 +10,7 @@ import { getShardIdFromUuid } from './sharding.js';
 import { RefManager } from './cow/RefManager.js';
 import { BlobStorage } from './cow/BlobStorage.js';
 import { CommitLog } from './cow/CommitLog.js';
+import { unwrapBinaryData, wrapBinaryData } from './cow/binaryDataCodec.js';
 import { prodLog } from '../utils/logger.js';
 // Clean directory structure (v4.7.2+)
 // All storage adapters use this consistent structure
@@ -234,32 +235,19 @@ export class BaseStorage extends BaseStorageAdapter {
                     if (data === null) {
                         return undefined;
                     }
-                    // Convert to Buffer
-                    if (Buffer.isBuffer(data)) {
-                        return data;
-                    }
-                    // v5.7.5: Unwrap binary data stored as {_binary: true, data: "base64..."}
+                    // v5.7.5/v5.10.1: Use shared binaryDataCodec utility (single source of truth)
+                    // Unwraps binary data stored as {_binary: true, data: "base64..."}
                     // Fixes "Blob integrity check failed" - hash must be calculated on original content
-                    if (data._binary && typeof data.data === 'string') {
-                        return Buffer.from(data.data, 'base64');
-                    }
-                    return Buffer.from(JSON.stringify(data));
+                    return unwrapBinaryData(data);
                 }
                 catch (error) {
                     return undefined;
                 }
             },
             put: async (key, data) => {
-                // Store as Buffer (for blob data) or parse JSON (for metadata)
-                let obj;
-                try {
-                    // Try to parse as JSON first (for metadata)
-                    obj = JSON.parse(data.toString());
-                }
-                catch {
-                    // Not JSON, store as binary (base64 encoded for JSON storage)
-                    obj = { _binary: true, data: data.toString('base64') };
-                }
+                // v5.10.1: Use shared binaryDataCodec utility (single source of truth)
+                // Wraps binary data or parses JSON for storage
+                const obj = wrapBinaryData(data);
                 await this.writeObjectToPath(`_cow/${key}`, obj);
             },
             delete: async (key) => {

package/dist/storage/cow/BlobStorage.js

@@ -15,6 +15,7 @@
  */
 import { createHash } from 'crypto';
 import { NULL_HASH, isNullHash } from './constants.js';
+import { unwrapBinaryData } from './binaryDataCodec.js';
 /**
  * State-of-the-art content-addressable blob storage
  *
@@ -207,7 +208,10 @@ export class BlobStorage {
             metadataBuffer = await this.adapter.get(`${tryPrefix}-meta:${hash}`);
             if (metadataBuffer) {
                 prefix = tryPrefix;
-                metadata = JSON.parse(metadataBuffer.toString());
+                // v5.10.1: Unwrap metadata before parsing (defense-in-depth)
+                // Metadata should be JSON, but adapter might return wrapped format
+                const unwrappedMetadata = unwrapBinaryData(metadataBuffer);
+                metadata = JSON.parse(unwrappedMetadata.toString());
                 break;
             }
         }
@@ -227,15 +231,20 @@ export class BlobStorage {
             }
             finalData = await this.zstdDecompress(data);
         }
-        // Verify hash (optional, expensive)
-        if (!options.skipVerification && BlobStorage.hash(finalData) !== hash) {
+        // v5.10.1: Defense-in-depth unwrap (CRITICAL FIX for blob integrity regression)
+        // Even though COW adapter should unwrap (v5.7.5), verify it happened and re-unwrap if needed
+        // This prevents "Blob integrity check failed" errors if adapter returns wrapped data
+        // Uses shared binaryDataCodec utility (single source of truth for unwrap logic)
+        const unwrappedData = unwrapBinaryData(finalData);
+        // Verify hash (on unwrapped data)
+        if (!options.skipVerification && BlobStorage.hash(unwrappedData) !== hash) {
             throw new Error(`Blob integrity check failed: ${hash}`);
         }
         // Add to cache (only if not skipped)
         if (!options.skipCache) {
-            this.addToCache(hash, finalData, metadata);
+            this.addToCache(hash, unwrappedData, metadata);
         }
-        return finalData;
+        return unwrappedData;
     }
     /**
      * Check if blob exists

package/dist/storage/cow/binaryDataCodec.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Binary Data Codec: Single Source of Truth for Wrap/Unwrap Operations
+ *
+ * This module provides the ONLY implementation of binary data encoding/decoding
+ * used across all storage adapters and blob storage.
+ *
+ * Design Principles:
+ * - DRY: One implementation, used everywhere
+ * - Single Responsibility: Only handles binary ↔ JSON conversion
+ * - Type-Safe: Proper TypeScript types
+ * - Defensive: Handles all edge cases
+ *
+ * Used by:
+ * - BaseStorage COW adapter (write/read operations)
+ * - BlobStorage (defense-in-depth verification)
+ * - All storage adapters (via BaseStorage)
+ *
+ * @module storage/cow/binaryDataCodec
+ */
+/**
+ * Wrapped binary data format
+ * Used when storing binary data in JSON-based storage
+ */
+export interface WrappedBinaryData {
+    _binary: true;
+    data: string;
+}
+/**
+ * Check if data is wrapped binary format
+ */
+export declare function isWrappedBinary(data: any): data is WrappedBinaryData;
+/**
+ * Unwrap binary data from JSON wrapper
+ *
+ * This is the SINGLE SOURCE OF TRUTH for unwrapping binary data.
+ * All storage operations MUST use this function.
+ *
+ * Handles:
+ * - Buffer → Buffer (pass-through)
+ * - {_binary: true, data: "base64..."} → Buffer (unwrap)
+ * - Plain object → Buffer (JSON stringify)
+ * - Other types → Error
+ *
+ * @param data - Data to unwrap (may be Buffer, wrapped object, or plain object)
+ * @returns Unwrapped Buffer
+ * @throws Error if data type is invalid
+ */
+export declare function unwrapBinaryData(data: any): Buffer;
+/**
+ * Wrap binary data for JSON storage
+ *
+ * This is the SINGLE SOURCE OF TRUTH for wrapping binary data.
+ * All storage operations MUST use this function.
+ *
+ * @param data - Buffer to wrap
+ * @returns Wrapped object or parsed JSON object
+ */
+export declare function wrapBinaryData(data: Buffer): any;
+/**
+ * Ensure data is a Buffer
+ *
+ * Convenience function that combines type checking and unwrapping.
+ * Use this when you need to ensure you have a Buffer.
+ *
+ * @param data - Data that should be or can be converted to Buffer
+ * @returns Buffer
+ */
+export declare function ensureBuffer(data: any): Buffer;

package/dist/storage/cow/binaryDataCodec.js

@@ -0,0 +1,103 @@
+/**
+ * Binary Data Codec: Single Source of Truth for Wrap/Unwrap Operations
+ *
+ * This module provides the ONLY implementation of binary data encoding/decoding
+ * used across all storage adapters and blob storage.
+ *
+ * Design Principles:
+ * - DRY: One implementation, used everywhere
+ * - Single Responsibility: Only handles binary ↔ JSON conversion
+ * - Type-Safe: Proper TypeScript types
+ * - Defensive: Handles all edge cases
+ *
+ * Used by:
+ * - BaseStorage COW adapter (write/read operations)
+ * - BlobStorage (defense-in-depth verification)
+ * - All storage adapters (via BaseStorage)
+ *
+ * @module storage/cow/binaryDataCodec
+ */
+/**
+ * Check if data is wrapped binary format
+ */
+export function isWrappedBinary(data) {
+    return (typeof data === 'object' &&
+        data !== null &&
+        data._binary === true &&
+        typeof data.data === 'string');
+}
+/**
+ * Unwrap binary data from JSON wrapper
+ *
+ * This is the SINGLE SOURCE OF TRUTH for unwrapping binary data.
+ * All storage operations MUST use this function.
+ *
+ * Handles:
+ * - Buffer → Buffer (pass-through)
+ * - {_binary: true, data: "base64..."} → Buffer (unwrap)
+ * - Plain object → Buffer (JSON stringify)
+ * - Other types → Error
+ *
+ * @param data - Data to unwrap (may be Buffer, wrapped object, or plain object)
+ * @returns Unwrapped Buffer
+ * @throws Error if data type is invalid
+ */
+export function unwrapBinaryData(data) {
+    // Case 1: Already a Buffer (no unwrapping needed)
+    if (Buffer.isBuffer(data)) {
+        return data;
+    }
+    // Case 2: Wrapped binary data {_binary: true, data: "base64..."}
+    if (isWrappedBinary(data)) {
+        return Buffer.from(data.data, 'base64');
+    }
+    // Case 3: Plain object (shouldn't happen for binary blobs, but handle gracefully)
+    if (typeof data === 'object' && data !== null) {
+        return Buffer.from(JSON.stringify(data));
+    }
+    // Case 4: String (convert to Buffer)
+    if (typeof data === 'string') {
+        return Buffer.from(data);
+    }
+    // Case 5: Invalid type
+    throw new Error(`Invalid data type for unwrap: ${typeof data}. ` +
+        `Expected Buffer or {_binary: true, data: "base64..."}`);
+}
+/**
+ * Wrap binary data for JSON storage
+ *
+ * This is the SINGLE SOURCE OF TRUTH for wrapping binary data.
+ * All storage operations MUST use this function.
+ *
+ * @param data - Buffer to wrap
+ * @returns Wrapped object or parsed JSON object
+ */
+export function wrapBinaryData(data) {
+    // Try to parse as JSON first (for metadata, trees, commits)
+    try {
+        return JSON.parse(data.toString());
+    }
+    catch {
+        // Not JSON - wrap as binary data
+        return {
+            _binary: true,
+            data: data.toString('base64')
+        };
+    }
+}
+/**
+ * Ensure data is a Buffer
+ *
+ * Convenience function that combines type checking and unwrapping.
+ * Use this when you need to ensure you have a Buffer.
+ *
+ * @param data - Data that should be or can be converted to Buffer
+ * @returns Buffer
+ */
+export function ensureBuffer(data) {
+    if (Buffer.isBuffer(data)) {
+        return data;
+    }
+    return unwrapBinaryData(data);
+}
+//# sourceMappingURL=binaryDataCodec.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@soulcraft/brainy",
-  "version": "5.10.0",
+  "version": "5.10.1",
   "description": "Universal Knowledge Protocol™ - World's first Triple Intelligence database unifying vector, graph, and document search in one API. Stage 3 CANONICAL: 42 nouns × 127 verbs covering 96-97% of all human knowledge.",
   "main": "dist/index.js",
   "module": "dist/index.js",