holepunch-hop 0.4.4 → 0.4.8

package/src/{bees.js → storage/bees.js}

@@ -22,6 +22,7 @@ class HyperBee extends EventEmitter {
     this.liveBees = {}
     this.confirmPubLibList = {}
     this.repPublicHolder = {}
+    this.activeBees = []
   }
 
   /**
@@ -113,6 +114,7 @@ class HyperBee extends EventEmitter {
       keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
       valueEncoding: 'json' // same options as above
     })
+
     await this.dbBentocues.ready()
     beePubkeys.push({store:'bentocues', privacy: 'public', pubkey: b4a.toString(core7.key, 'hex')})
     // open the cues library
@@ -181,40 +183,7 @@ class HyperBee extends EventEmitter {
     startBeePubkey.data = beePubkeys
     this.liveBees = startBeePubkey
     this.wsocket.send(JSON.stringify(startBeePubkey))
-  }
-
-  /**
-   * save pair in keystore public network library
-   * @method savePubliclibrary
-   *
-  */
-  savePubliclibrary = async function (refContract) {
-    let beeSave = await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
-    // go query the key are return the info. to ensure data save asplanned.
-    let saveCheck = await this.getPublicLibrary(refContract.data.hash)
-    let returnMessage = {}
-    returnMessage.stored = true
-    returnMessage.type = refContract.reftype
-    returnMessage.key = saveCheck.key
-    returnMessage.contract = saveCheck.value
-    return returnMessage
-  }
-  
-
-  /**
-   * save pair in keystore db
-   * @method savePeerLibrary
-   *
-  */
-  savePeerLibrary = async function (refContract) {
-    await this.dbPeerLibrary.put(refContract.data.hash, refContract.data.contract)
-    let saveCheck = await this.getPeerLibrary(refContract.data.hash)
-    let returnMessage = {}
-    returnMessage.stored = true
-    returnMessage.type = refContract.reftype
-    returnMessage.key = saveCheck.key
-    returnMessage.contract = saveCheck.value
-    return returnMessage
+    this.activeBees = beePubkeys
   }
 
   /**
@@ -698,6 +667,42 @@ class HyperBee extends EventEmitter {
     return nodeData
   }
 
+ //** */ public library **//
+
+  /**
+   * save pair in keystore public network library
+   * @method savePubliclibrary
+   *
+  */
+  savePubliclibrary = async function (refContract) {
+    await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
+    // go query the key are return the info. to ensure data save asplanned.
+    let saveCheck = await this.getPublicLibrary(refContract.data.hash)
+    let returnMessage = {}
+    returnMessage.stored = true
+    returnMessage.type = refContract.reftype
+    returnMessage.key = saveCheck.key
+    returnMessage.contract = saveCheck.value
+    return returnMessage
+  }
+  
+
+  /**
+   * save pair in keystore db
+   * @method savePeerLibrary
+   *
+  */
+  savePeerLibrary = async function (refContract) {
+    await this.dbPeerLibrary.put(refContract.data.hash, refContract.data.contract)
+    let saveCheck = await this.getPeerLibrary(refContract.data.hash)
+    let returnMessage = {}
+    returnMessage.stored = true
+    returnMessage.type = refContract.reftype
+    returnMessage.key = saveCheck.key
+    returnMessage.contract = saveCheck.value
+    return returnMessage
+  }
+
   /**
    * lookup specific result UUID
    * @method getPublicLibrary
@@ -890,9 +895,14 @@ class HyperBee extends EventEmitter {
     for await (const { key, value } of nodeData) {
       resData.push({ key, value })
     }
-    this.repPublicHolder[dataIn.discoverykey] = resData
-    // notify peer repliate complete, ask if want save
-    this.emit('publib-replicate-notification', { data: { text: 'public library replication complete', publib: dataIn.discoverykey }})
+    // data received?
+    if (resData.length > 0) {
+      this.repPublicHolder[dataIn.discoverykey] = resData
+      // notify peer repliate complete, ask if want save
+      this.emit('publib-replicate-notification', { data: { text: 'public library replication complete', publib: dataIn.discoverykey }})
+    } else {
+      this.emit('publib-replicate-notification', { data: { text: 'no data received', publib: dataIn.discoverykey }})
+    }
   }
 
   /**

package/src/{drive.js → storage/drive.js}

@@ -13,7 +13,7 @@ import fs from 'fs'
 import Hyperdrive from 'hyperdrive'
 import b4a from 'b4a'
 import Fileparser from './fileParser.js'
-import SqliteAdapter from './adapters/sqliteDatabase.js'
+import SqliteAdapter from '../adapters/sqliteDatabase.js'
 import csv from 'csv-parser'
 import { DateTime, Interval } from 'luxon'
 import * as chrono from 'chrono-node'
@@ -54,12 +54,11 @@ class HypDrive extends EventEmitter {
     // One for a file metadata Hyperbee, and one for a content Hypercore
     this.drive = new Hyperdrive(this.core)
     await this.drive.ready()
-
     let startDrivePubkey = {}
     startDrivePubkey.type = 'account'
     startDrivePubkey.action = 'drive-pubkey'
     startDrivePubkey.data =  b4a.toString(this.drive.key, 'hex')
-    this.wsocket.send(JSON.stringify(startDrivePubkey))
+    // this.wsocket.send(JSON.stringify(startDrivePubkey))
   }
 
 

package/src/{fileParser.js → storage/fileParser.js}

@@ -18,7 +18,7 @@ import events from 'events'
 import axios from 'axios'
 import csv from 'csv-parser'
 import crypto from 'crypto'
-import TimeConvert from './adapters/timeConvertor.js'
+import TimeConvert from '../adapters/timeConvertor.js'
 import { DateTime, Interval } from 'luxon'
 import * as chrono from 'chrono-node'
 

package/test/datacommands/files/data/jan3-bitcoin.csv

@@ -0,0 +1,17 @@
+datetime,tz_cd,price,volume,source,site_no
+2009-01-03 01:30,PST,0,0,bitcoin,0
+2010-01-03 01:30,PST,0,0,bitcoin,0
+2011-01-03 01:30,PST,0.3,14300,bitcoin,0
+2012-01-03 01:30,PST,4.9,125170,bitcoin,0
+2013-01-03 01:30,PST,13.4,18030,bitcoin,0
+2014-01-03 01:30,PST,884.3,9710,bitcoin,0
+2015-01-03 01:30,PST,287.1,47370,bitcoin,0
+2016-01-03 01:30,PST,430.7,54830,bitcoin,0
+2017-01-03 01:30,PST,1033.3,547900,bitcoin,0
+2018-01-03 01:31,PST,15156.6,106540,bitcoin,0
+2019-01-03 01:32,PST,3780.1,468450,bitcoin,0
+2020-01-03 01:33,PST,7343.1,938290,bitcoin,0
+2021-01-03 01:34,PST,32958.9,155210,bitcoin,0
+2022-01-03 01:35,PST,46430.2,41060,bitcoin,0
+2023-01-03 01:36,PST,16674.2,178730,bitcoin,0
+2024-01-03 01:37,PST,42836.1,117650,bitcoin,0

package/test/datacommands/files/large-csv.test.js

@@ -0,0 +1,7 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+
+describe('Large CSV File Operations Tests', () => {
+  it.skip('Large CSV file operations', () => {
+    // This test is skipped for now
+  })
+})

package/test/datacommands/files/small-csv.test.js

@@ -0,0 +1,76 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import Corestore from 'corestore'
+import fs from 'fs'
+import path from 'path'
+import os from 'os'
+import Hyperswarm from 'hyperswarm'
+import goodbye from 'graceful-goodbye'
+import b4a from 'b4a'
+import DriveWorker from '../../../src/storage/drive.js'
+
+// Get the path to the test data directory
+const testDataPath = path.join(__dirname, 'data')
+const testCSVPath = path.join(testDataPath, 'jan3-bitcoin.csv')
+
+describe('CSV File Operations Tests', () => {
+  let store
+  let swarm
+  let clientDrive
+
+  beforeEach(async () => {
+
+    store = new Corestore(os.homedir() + '/.test-file-store')
+    swarm = new Hyperswarm()
+    // make replication possible
+    swarm.on('connection', conn => store.replicate(conn))
+    goodbye(() => swarm.destroy())
+
+    // Create HypDrive instance
+    clientDrive = new DriveWorker(store, swarm)
+
+    // Initialize drive
+    await clientDrive.setupHyperdrive()
+  })
+
+  afterEach(async () => {
+    // Clean up
+  })
+
+  it('should save and retrieve small CSV file', async () => {
+    // 1. Read the actual CSV file
+    const csvContent = fs.readFileSync(testCSVPath, 'utf-8')
+    const csvFileName = path.basename(testCSVPath)
+
+    // 2. Save CSV file
+    await clientDrive.hyperdriveFilesave('text/csv', csvFileName, csvContent)
+
+    // 3. Verify file is saved
+    const clientFileContent = await clientDrive.drive.get('text/csv/' + csvFileName)
+    expect(clientFileContent.toString()).toBe(csvContent)
+
+    // 4. Verify file metadata
+    const fileInfo = await clientDrive.drive.entry('text/csv/' + csvFileName)
+    expect(fileInfo).toBeDefined()
+    expect(fileInfo.value.blob).toBeDefined()
+    expect(fileInfo.value.blob.byteLength).toBe(csvContent.length)
+
+    // 5. Verify file structure (headers)
+    const fileLines = csvContent.split('\n')
+    const headers = fileLines[0].split(',')
+    expect(headers.length).toBeGreaterThan(0)
+    expect(headers[0]).toBeDefined()
+
+    // 6. Verify specific 2015 entry
+    const entry2015 = fileLines[7] // 7th line (0-indexed)
+    expect(entry2015).toBe('2015-01-03 01:30,PST,287.1,47370,bitcoin,0')
+
+    // 7. Verify specific values in 2015 entry
+    const entryValues = entry2015.split(',')
+    expect(entryValues[0]).toBe('2015-01-03 01:30')
+    expect(entryValues[1]).toBe('PST')
+    expect(entryValues[2]).toBe('287.1')
+    expect(entryValues[3]).toBe('47370')
+    expect(entryValues[4]).toBe('bitcoin')
+    expect(entryValues[5]).toBe('0')
+  })
+})

package/test/datacommands/ledger/save-get-ledger.test.js

@@ -0,0 +1,117 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import Corestore from 'corestore'
+import Hyperswarm from 'hyperswarm'
+import Hyperbee from 'hyperbee'
+import b4a from 'b4a'
+import hashObject from 'object-hash'
+import path from 'path'
+import os from 'os'
+import goodbye from 'graceful-goodbye'
+import BeeWorker from '../../../src/storage/bees.js'
+
+// Get the path to the test data directory
+const testDataPath = path.join(__dirname, 'data')
+
+describe('Hyperbee Ledger Tests', () => {
+  let store
+  let swarm
+  let bee
+
+  beforeAll(async () => {
+    console.log('start of before each')
+    store = new Corestore(os.homedir() + '/.test-ledger-store')
+    swarm = new Hyperswarm()
+    // make replication possible
+    swarm.on('connection', conn => store.replicate(conn))
+    goodbye(() => swarm.destroy())
+
+    // Additional wait for full initialization
+    await new Promise((resolve) => setTimeout(resolve, 4000))
+    console.log('wait afoter')
+    // Create BeeWorker instance
+    bee = new BeeWorker(store, swarm)
+    // await bee.setupHyperbee()  vitest have issues starting so many core at once
+    const core5 = store.get({ name: 'kbledger' })
+    bee.dbKBledger = new Hyperbee(core5, {
+      keyEncoding: 'utf-8',
+      valueEncoding: 'json'
+    })
+    await bee.dbKBledger.ready()
+    await new Promise((resolve) => setTimeout(resolve, 2000))
+  })
+
+  afterAll(async () => {
+    console.log('Cleaning up test environment')
+    // await swarm.destroy()
+    await bee.dbKBledger.close()
+    bee = null
+    store = null
+    // swarm = null
+  }, 20000) // 20 second timeout for cleanup
+
+  it('should perform basic Hyperbee operations', async () => {
+    // Test data
+    console.log('start of test')
+    const testHash = 'test-hash'
+    const testData = { message: 'Hello Hyperbee!' }
+
+    // 1. Put data
+    await bee.dbKBledger.put(testHash, testData)
+    
+    // 2. Get data
+    const result = await bee.dbKBledger.get(testHash)
+    expect(result.value).toEqual(testData)
+
+    // 3. Batch operations with 32-byte keys
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const batch = bee.dbKBledger.batch()
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+    await batch.put(key1, res1)
+    await batch.put(key2, res2)
+    await batch.flush()
+
+    // 4. CreateReadStream
+    const stream = bee.dbKBledger.createReadStream()
+    const entries = []
+    for await (const entry of stream) {
+      entries.push(entry)
+    }
+    expect(entries.length).toBeGreaterThan(0)
+
+    // 5. Check version
+    expect(bee.dbKBledger.version).toBeGreaterThan(0)
+
+    // 6. Verify batch operations
+    const result1 = await bee.dbKBledger.get(key1)
+    expect(result1.key).toEqual(key1)
+    const result2 = await bee.dbKBledger.get(key2)
+    expect(result2.key).toEqual(key2)
+  })
+
+  it('should handle key value operations', async () => {
+    // Test data
+    console.log('start of test2')
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+
+    // Put multiple values
+    await bee.dbKBledger.put(key1, res1)
+    await bee.dbKBledger.put(key2, res2)
+
+    // Get values
+    const result1 = await bee.dbKBledger.get(key1)
+    const result2 = await bee.dbKBledger.get(key2)
+    expect(result1.value.data.value[0]).toEqual(1)
+    expect(result2.value.data.value[0]).toEqual(4)
+
+    // Delete value
+    await bee.dbKBledger.del(key1)
+    const deletedResult = await bee.dbKBledger.get(key1)
+    expect(deletedResult).toBeNull()
+    console.log('end of test2')
+  })
+})

package/test/datacommands/results/save-get-results.test.js

@@ -0,0 +1,122 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import Corestore from 'corestore'
+import Hyperswarm from 'hyperswarm'
+import Hyperbee from 'hyperbee'
+import b4a from 'b4a'
+import hashObject from 'object-hash'
+import path from 'path'
+import os from 'os'
+import goodbye from 'graceful-goodbye'
+import BeeWorker from '../../../src/storage/bees.js'
+
+// Get the path to the test data directory
+const testDataPath = path.join(__dirname, 'data')
+
+describe('Hyperbee Operations Tests', () => {
+  let store
+  let swarm
+  let bee
+  let core4
+
+  beforeAll(async () => {
+    console.log('start of before each')
+    store = new Corestore(os.homedir() + '/.test-bee-store')
+    swarm = new Hyperswarm()
+    // make replication possible
+    swarm.on('connection', conn => store.replicate(conn))
+    goodbye(() => swarm.destroy())
+
+    // Additional wait for full initialization
+    await new Promise((resolve) => setTimeout(resolve, 4000))
+    console.log('wait afoter')
+    // Create BeeWorker instance
+    bee = new BeeWorker(store, swarm)
+    // await bee.setupHyperbee()  vitest have issues starting so many core at once
+    core4 = store.get({ name: 'hopresults' })
+    bee.dbHOPresults = new Hyperbee(core4, {
+      keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
+      valueEncoding: 'json' // same options as above
+    })
+    await bee.dbHOPresults.ready()
+    await new Promise((resolve) => setTimeout(resolve, 2000))
+
+  })
+
+  afterAll(async () => {
+    console.log('Cleaning up test environment')
+    // await swarm.destroy()
+    await bee.dbHOPresults.close()
+    await core4.close()
+    bee = null
+    store = null
+    //swarm = null
+    core4 = null
+  }, 20000) // 20 second timeout for cleanup
+
+  it('should perform basic Hyperbee operations', async () => {
+    // Test data
+    console.log('start of test')
+    const testHash = 'test-hash'
+    const testData = { message: 'Hello Hyperbee!' }
+
+    // 1. Put data
+    await bee.dbHOPresults.put(testHash, testData)
+    
+    // 2. Get data
+    const result = await bee.dbHOPresults.get(testHash)
+    expect(result.value).toEqual(testData)
+
+    // 3. Batch operations with 32-byte keys
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const batch = bee.dbHOPresults.batch()
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+    await batch.put(key1, res1)
+    await batch.put(key2, res2)
+    await batch.flush()
+
+    // 4. CreateReadStream
+    const stream = bee.dbHOPresults.createReadStream()
+    const entries = []
+    for await (const entry of stream) {
+      entries.push(entry)
+    }
+    expect(entries.length).toBeGreaterThan(0)
+
+    // 5. Check version
+    expect(bee.dbHOPresults.version).toBeGreaterThan(0)
+
+    // 6. Verify batch operations
+    const result1 = await bee.dbHOPresults.get(key1)
+    expect(result1.key).toEqual(key1)
+    const result2 = await bee.dbHOPresults.get(key2)
+    expect(result2.key).toEqual(key2)
+    
+  })
+
+  it('should handle key value operations', async () => {
+    // Test data
+    console.log('start of test2')
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+
+    // Put multiple values
+    await bee.dbHOPresults.put(key1, res1)
+    await bee.dbHOPresults.put(key2, res2)
+
+    // Get values
+    const result1 = await bee.dbHOPresults.get(key1)
+    const result2 = await bee.dbHOPresults.get(key2)
+    expect(result1.value.data.value[0]).toEqual(1)
+    expect(result2.value.data.value[0]).toEqual(4)
+
+    // Delete value
+    await bee.dbHOPresults.del(key1)
+    const deletedResult = await bee.dbHOPresults.get(key1)
+    expect(deletedResult).toBeNull()
+    console.log('end of test2')
+  })
+})

package/test/holepunch-initiate.test.js

@@ -0,0 +1,47 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import { spawn } from 'child_process'
+import path from 'path'
+import HolepunchWorker from '../src/index.js'
+
+// Set global test timeout to 10 seconds
+const testTimeout = 10000;
+
+describe('holepunch initialization', () => {
+  let hopProcess
+  let holepunch
+
+  beforeAll(async () => {
+    // Start HOP server
+    const baseHOPStepsUp = path.join(__dirname, '..')
+    hopProcess = spawn('npm', ['run', 'start'], { stdio: 'inherit', cwd: baseHOPStepsUp })
+
+    // Wait for server to start
+    await new Promise((resolve) => setTimeout(resolve, 3000))
+
+    // Initialize HolepunchWorker
+    holepunch = new HolepunchWorker('.test-instance')
+    await holepunch.startHolepunch()
+    // Additional wait for full initialization
+    await new Promise((resolve) => setTimeout(resolve, 2000))
+  })
+
+  afterAll(async () => {
+    // Clean up HolepunchWorker
+    // Stop HOP server
+    if (hopProcess) {
+      hopProcess.kill()
+    }
+  })
+
+  it('should have initialized store', () => {
+    expect(holepunch.store).toBeDefined()
+  })
+
+  it('should have initialized swarm', () => {
+    expect(holepunch.swarm).toBeDefined()
+  })
+
+  it('should have initialized BeeData', () => {
+    expect(holepunch.BeeData).toBeDefined()
+  })
+})