@bsv/sdk 1.6.26 → 1.7.0

package/docs/reference/storage.md

@@ -22,7 +22,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](
 
 ```ts
 export interface DownloadResult {
-    data: number[];
+    data: Uint8Array;
     mimeType: string | null;
 }
 ```
@@ -85,7 +85,7 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](
 
 ```ts
 export interface UploadableFile {
-    data: number[];
+    data: Uint8Array | number[];
     type: string;
 }
 ```
@@ -341,13 +341,20 @@ Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](
 ### Variable: getURLForFile
 
 ```ts
-getURLForFile = (file: number[]): string => {
-    const hash = sha256(file);
+getURLForFile = (file: Uint8Array | number[]): string => {
+    const data = file instanceof Uint8Array ? file : Uint8Array.from(file);
+    const hasher = new Hash.SHA256();
+    const chunkSize = 1024 * 1024;
+    for (let i = 0; i < data.length; i += chunkSize) {
+        const chunk = data.subarray(i, i + chunkSize);
+        hasher.update(Array.from(chunk));
+    }
+    const hash = hasher.digest();
     return getURLForHash(hash);
 }
 ```
 
-See also: [getURLForHash](./storage.md#variable-geturlforhash), [sha256](./primitives.md#variable-sha256)
+See also: [getURLForHash](./storage.md#variable-geturlforhash)
 
 Links: [API](#api), [Interfaces](#interfaces), [Classes](#classes), [Functions](#functions), [Types](#types), [Enums](#enums), [Variables](#variables)
 

package/docs/tutorials/uhrp-storage.md

@@ -71,7 +71,7 @@ async function basicFileUpload() {
   // Create sample file
   const fileData = new TextEncoder().encode('Hello, UHRP storage!')
   const file = {
-    data: Array.from(fileData),
+    data: fileData,
     type: 'text/plain'
   }
   
@@ -116,7 +116,7 @@ async function basicFileDownload(uhrpUrl: string) {
     
     // Convert to string if text file
     if (result.mimeType?.startsWith('text/')) {
-      const content = new TextDecoder().decode(new Uint8Array(result.data))
+      const content = new TextDecoder().decode(result.data)
       console.log('Content:', content)
     }
     
@@ -172,7 +172,7 @@ class UHRPFileManager {
     tags: string[] = []
   ): Promise<FileMetadata> {
     const file = {
-      data: Array.from(fileData),
+      data: fileData,
       type: mimeType
     }
     
@@ -217,7 +217,7 @@ class UHRPFileManager {
       console.log('File downloaded:', uhrpUrl)
       
       return {
-        data: new Uint8Array(result.data),
+        data: result.data,
         metadata
       }
     } catch (error) {
@@ -371,7 +371,7 @@ class BatchFileOperations {
     const results = await Promise.allSettled(
       files.map(async (file) => {
         const fileObj = {
-          data: Array.from(file.data),
+          data: file.data,
           type: file.type
         }
         
@@ -422,7 +422,7 @@ class BatchFileOperations {
         return {
           success: true,
           url,
-          data: new Uint8Array(result.value.data)
+          data: result.value.data
         }
       } else {
         return {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bsv/sdk",
-  "version": "1.6.26",
+  "version": "1.7.0",
   "type": "module",
   "description": "BSV Blockchain Software Development Kit",
   "main": "dist/cjs/mod.js",

package/src/storage/StorageDownloader.ts

@@ -9,7 +9,7 @@ export interface DownloaderConfig {
 }
 
 export interface DownloadResult {
-  data: number[]
+  data: Uint8Array
   mimeType: string | null
 }
 
@@ -58,6 +58,7 @@ export class StorageDownloader {
       throw new Error('Invalid parameter UHRP url')
     }
     const hash = StorageUtils.getHashFromURL(uhrpUrl)
+    const expected = Utils.toHex(hash)
     const downloadURLs = await this.resolve(uhrpUrl)
 
     if (!Array.isArray(downloadURLs) || downloadURLs.length === 0) {
@@ -70,22 +71,37 @@ export class StorageDownloader {
         const result = await fetch(downloadURLs[i], { method: 'GET' })
 
         // If the request fails, continue to the next url
-        if (!result.ok || result.status >= 400) {
+        if (!result.ok || result.status >= 400 || result.body == null) {
           continue
         }
-        const body = await result.arrayBuffer()
-
-        // The body is loaded into a number array
-        const content: number[] = [...new Uint8Array(body)]
-        const contentHash = Hash.sha256(content)
-        for (let i = 0; i < contentHash.length; ++i) {
-          if (contentHash[i] !== hash[i]) {
-            throw new Error('Value of content does not match hash of the url given')
-          }
+
+        const reader = result.body.getReader()
+        const hashStream = new Hash.SHA256()
+        const chunks: Uint8Array[] = []
+        let totalLength = 0
+
+        while (true) {
+          const { done, value } = await reader.read()
+          if (done) break
+          hashStream.update(Array.from(value))
+          chunks.push(value)
+          totalLength += value.length
+        }
+
+        const digest = Utils.toHex(hashStream.digest())
+        if (digest !== expected) {
+          throw new Error('Data integrity error: value of content does not match hash of the url given')
+        }
+
+        const data = new Uint8Array(totalLength)
+        let offset = 0
+        for (const chunk of chunks) {
+          data.set(chunk, offset)
+          offset += chunk.length
         }
 
         return {
-          data: content,
+          data,
           mimeType: result.headers.get('Content-Type')
         }
       } catch (error) {

package/src/storage/StorageUploader.ts

@@ -8,7 +8,7 @@ export interface UploaderConfig {
 }
 
 export interface UploadableFile {
-  data: number[]
+  data: Uint8Array | number[]
   type: string
 }
 
@@ -108,7 +108,7 @@ export class StorageUploader {
     file: UploadableFile,
     requiredHeaders: Record<string, string>
   ): Promise<UploadFileResult> {
-    const body = Uint8Array.from(file.data)
+    const body = file.data instanceof Uint8Array ? file.data : Uint8Array.from(file.data)
 
     const response = await fetch(uploadURL, {
       method: 'PUT',
@@ -122,7 +122,7 @@ export class StorageUploader {
       throw new Error(`File upload failed: HTTP ${response.status}`)
     }
 
-    const uhrpURL = await StorageUtils.getURLForFile(file.data)
+    const uhrpURL = StorageUtils.getURLForFile(body)
     return {
       published: true,
       uhrpURL
@@ -148,10 +148,11 @@ export class StorageUploader {
     retentionPeriod: number
   }): Promise<UploadFileResult> {
     const { file, retentionPeriod } = params
-    const fileSize = file.data.length
+    const data = file.data instanceof Uint8Array ? file.data : Uint8Array.from(file.data)
+    const fileSize = data.byteLength
 
     const { uploadURL, requiredHeaders } = await this.getUploadInfo(fileSize, retentionPeriod)
-    return await this.uploadFile(uploadURL, file, requiredHeaders)
+    return await this.uploadFile(uploadURL, { data, type: file.type }, requiredHeaders)
   }
 
   /**

package/src/storage/StorageUtils.ts

@@ -1,5 +1,5 @@
-import { sha256 } from '../primitives/Hash.js'
 import { toHex, fromBase58Check, toBase58Check, toArray } from '../primitives/utils.js'
+import { Hash } from '../primitives/index.js'
 
 /**
  * Takes a UHRP URL and removes any prefixes.
@@ -26,11 +26,19 @@ export const getURLForHash = (hash: number[]): string => {
 
 /**
  * Generates a UHRP URL for a given file.
- * @param {number[] | string} file - File content as number array or string.
+ * Uses a streaming hash to avoid excessive memory usage with large files.
+ * @param {Uint8Array | number[]} file - File content as a typed array or number array.
  * @returns {string} - Base58Check encoded URL.
  */
-export const getURLForFile = (file: number[]): string => {
-  const hash = sha256(file)
+export const getURLForFile = (file: Uint8Array | number[]): string => {
+  const data = file instanceof Uint8Array ? file : Uint8Array.from(file)
+  const hasher = new Hash.SHA256()
+  const chunkSize = 1024 * 1024
+  for (let i = 0; i < data.length; i += chunkSize) {
+    const chunk = data.subarray(i, i + chunkSize)
+    hasher.update(Array.from(chunk))
+  }
+  const hash = hasher.digest()
   return getURLForHash(hash)
 }
 

package/src/storage/__tests/StorageDownloader.test.ts

@@ -3,8 +3,9 @@ import { StorageUtils } from '../index.js'
 import { LookupResolver } from '../../overlay-tools/index.js'
 import Transaction from '../../transaction/Transaction.js'
 import PushDrop from '../../script/templates/PushDrop.js'
-import { Hash, PublicKey } from '../../primitives/index.js'
+import { PublicKey } from '../../primitives/index.js'
 import { Utils } from '../../primitives/index.js'
+import { ReadableStream } from 'stream/web'
 
 beforeEach(() => {
     jest.restoreAllMocks()
@@ -125,7 +126,7 @@ describe('StorageDownloader', () => {
             const result = await downloader.download('validUrl')
             expect(fetchSpy).toHaveBeenCalledTimes(2)
             expect(result).toEqual({
-                data: new Array(32).fill(0),
+                data: new Uint8Array(32).fill(0),
                 mimeType: 'application/test'
             })
         })
@@ -200,5 +201,36 @@ describe('StorageDownloader', () => {
                 .resolves
                 .toEqual(["", ""])
         })
+
+        it.skip('downloads and verifies large streamed content', async () => {
+            const size = 5 * 1024 * 1024
+            const data = new Uint8Array(size)
+            for (let i = 0; i < size; i++) data[i] = i % 256
+            const uhrpUrl = StorageUtils.getURLForFile(data)
+
+            jest.spyOn(downloader, 'resolve').mockResolvedValue(['http://large-file'])
+
+            const chunkSize = 64 * 1024
+            const stream = new ReadableStream<Uint8Array>({
+                start (controller) {
+                    for (let offset = 0; offset < data.length; offset += chunkSize) {
+                        controller.enqueue(data.subarray(offset, offset + chunkSize))
+                    }
+                    controller.close()
+                }
+            })
+
+            jest.spyOn(global, 'fetch').mockResolvedValue(
+                new Response(stream as any, {
+                    status: 200,
+                    headers: { 'Content-Type': 'application/octet-stream' }
+                })
+            )
+
+            const result = await downloader.download(uhrpUrl)
+            expect(result.mimeType).toBe('application/octet-stream')
+            expect(result.data.length).toBe(size)
+            expect(result.data).toEqual(data)
+        })
     })
 })

package/src/storage/__tests/StorageUploader.test.ts

@@ -1,6 +1,7 @@
 import { StorageUploader } from '../StorageUploader.js'
 import * as StorageUtils from '../StorageUtils.js'
 import WalletClient from '../../wallet/WalletClient.js'
+import { createHash } from 'crypto'
 
 /**
  * A helper for converting a string to a number[] of UTF-8 bytes
@@ -66,6 +67,25 @@ describe('StorageUploader Tests', () => {
     expect(firstFour).toHaveLength(8)
   })
 
+  it('should handle large file uploads efficiently', async () => {
+    const size = 5 * 1024 * 1024
+    const data = new Uint8Array(size)
+    for (let i = 0; i < size; i++) data[i] = i % 256
+
+    jest.spyOn(uploader as any, 'getUploadInfo').mockResolvedValue({
+      uploadURL: 'https://example-upload.com/put'
+    })
+
+    const result = await uploader.publishFile({
+      file: { data, type: 'application/octet-stream' },
+      retentionPeriod: 7
+    })
+
+    const expectedHash = createHash('sha256').update(data).digest()
+    const urlHash = StorageUtils.getHashFromURL(result.uhrpURL)
+    expect(Buffer.from(urlHash)).toEqual(expectedHash)
+  })
+
   it('should throw if the upload fails with HTTP 500', async () => {
     // Force the direct upload (global fetch) to fail
     globalFetchSpy.mockResolvedValueOnce(new Response(null, { status: 500 }))