holepunch-hop 0.4.5 → 0.5.0

package/src/{bees.js → storage/bees.js}

@@ -22,6 +22,7 @@ class HyperBee extends EventEmitter {
     this.liveBees = {}
     this.confirmPubLibList = {}
     this.repPublicHolder = {}
+    this.activeBees = []
   }
 
   /**
@@ -86,7 +87,7 @@ class HyperBee extends EventEmitter {
       valueEncoding: 'json' // same options as above
     })
     await this.dbBentochat.ready()
-    beePubkeys.push({store:'bentochat', privacy: 'private', pubkey: b4a.toString(core3.key, 'hex')})
+    beePubkeys.push({store:'bentochat', privacy: 'private', pubkey: b4a.toString(core14.key, 'hex')})
 
 
     const core4 = this.store.get({ name: 'hopresults' })
@@ -113,6 +114,7 @@ class HyperBee extends EventEmitter {
       keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
       valueEncoding: 'json' // same options as above
     })
+
     await this.dbBentocues.ready()
     beePubkeys.push({store:'bentocues', privacy: 'public', pubkey: b4a.toString(core7.key, 'hex')})
     // open the cues library
@@ -128,7 +130,15 @@ class HyperBee extends EventEmitter {
       valueEncoding: 'json' // same options as above
     })
     await this.dbBentomodels.ready()
-    beePubkeys.push({store:'bentomodels', privacy: 'public', pubkey: b4a.toString(core7.key, 'hex')})
+    beePubkeys.push({store:'bentomodels', privacy: 'public', pubkey: b4a.toString(core13.key, 'hex')})
+
+    const core15 = this.store.get({ name: 'bentoboxes' })
+    this.dbBentoBoxes = new Hyperbee(core15, {
+      keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
+      valueEncoding: 'json' // same options as above
+    })
+    await this.dbBentoBoxes.ready()
+    beePubkeys.push({store:'bentoboxes', privacy: 'private', pubkey: b4a.toString(core15.key, 'hex')})
 
     const core8 = this.store.get({ name: 'bentodecisions' })
     this.dbBentodecisions = new Hyperbee(core8, {
@@ -138,7 +148,6 @@ class HyperBee extends EventEmitter {
     await this.dbBentodecisions.ready()
     beePubkeys.push({store:'bentodecisions', pubkey: b4a.toString(core8.key, 'hex')})
 
-
     const core9 = this.store.get({ name: 'bentomarkers' })
     this.dbBentomarkers = new Hyperbee(core9, {
       keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
@@ -147,7 +156,6 @@ class HyperBee extends EventEmitter {
     await this.dbBentomarkers.ready()
     beePubkeys.push({store:'bentomarkers', privacy: 'private', pubkey: b4a.toString(core9.key, 'hex')})
 
-
     const core10 = this.store.get({ name: 'research' })
     this.dbBentoresearch = new Hyperbee(core10, {
       keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
@@ -156,7 +164,6 @@ class HyperBee extends EventEmitter {
     await this.dbBentoresearch.ready()
     beePubkeys.push({store:'research', privacy: 'public', pubkey: b4a.toString(core10.key, 'hex')})
 
-
     const core11 = this.store.get({ name: 'bentoproducts' })
     this.dbBentoproducts = new Hyperbee(core11, {
       keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
@@ -181,60 +188,57 @@ class HyperBee extends EventEmitter {
     startBeePubkey.data = beePubkeys
     this.liveBees = startBeePubkey
     this.wsocket.send(JSON.stringify(startBeePubkey))
+    this.activeBees = beePubkeys
+
+    // test list of results
+    /* let listResults = await this.peerResults()
+    console.log(listResults)
+    for (let res of listResults) {
+      await this.deleteResultsItem(res.key)
+    } */
   }
 
+  /* HOP query results */
+
   /**
-   * save pair in keystore public network library
-   * @method savePubliclibrary
+   * save HOPresults
+   * @method saveHOPresults
    *
   */
-  savePubliclibrary = async function (refContract) {
-    let beeSave = await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
-    // go query the key are return the info. to ensure data save asplanned.
-    let saveCheck = await this.getPublicLibrary(refContract.data.hash)
-    let returnMessage = {}
-    returnMessage.stored = true
-    returnMessage.type = refContract.reftype
-    returnMessage.key = saveCheck.key
-    returnMessage.contract = saveCheck.value
-    return returnMessage
+  saveHOPresults = async function (refContract) {
+    await this.dbHOPresults.put(refContract.hash, refContract.data)
   }
-  
 
-  /**
-   * save pair in keystore db
-   * @method savePeerLibrary
-   *
-  */
-  savePeerLibrary = async function (refContract) {
-    await this.dbPeerLibrary.put(refContract.data.hash, refContract.data.contract)
-    let saveCheck = await this.getPeerLibrary(refContract.data.hash)
-    let returnMessage = {}
-    returnMessage.stored = true
-    returnMessage.type = refContract.reftype
-    returnMessage.key = saveCheck.key
-    returnMessage.contract = saveCheck.value
-    return returnMessage
+  peerResults = async function () {
+    const nodeData = this.dbHOPresults.createReadStream()
+    let resData = []
+    for await (const { key, value } of nodeData) {
+      resData.push({ key, value })
+    }
+    return resData
   }
 
   /**
-   * save kbledger entry
-   * @method saveKBLentry
+   * lookup results per dataprint hash
+   * @method peerResultsItem
    *
   */
-  saveKBLentry = async function (ledgerEntry) {
-    await this.dbKBledger.put(ledgerEntry.hash, ledgerEntry.data)
+  peerResultsItem = async function (key) {
+    const resultData = await this.dbHOPresults.get(key)
+    return resultData
   }
 
   /**
-   * save HOPresults
-   * @method saveHOPresults
+   * delete results
+   * @method deleteResultsItem
    *
   */
-  saveHOPresults = async function (refContract) {
-    await this.dbHOPresults.put(refContract.hash, refContract.data)
+  deleteResultsItem = async function (key) {
+    const resultData = await this.dbHOPresults.del(key)
+    return resultData
   }
 
+
   /** CHAT */
 
   /**
@@ -392,6 +396,53 @@ class HyperBee extends EventEmitter {
     return deleteInfo
   }
 
+  /** BENTOBOXES */
+  /**
+   * save model
+   * @method saveModel
+  */
+  saveBentoBox = async function (boxInfo) {
+    await this.dbBentoBoxes.put(boxInfo.id, boxInfo.data)
+    let checkSave = await this.getBentoBox(boxInfo.id)
+    return checkSave
+  }
+
+  /**
+   * get one cue by id
+   * @method getModel
+   *
+  */
+  getBentoBox = async function (key) {
+    const nodeData = await this.dbBentoBoxes.get(key)
+    return nodeData
+  }
+
+  /**
+   * get all cuees
+   * @method getModelHistory
+   *
+  */
+  getBentoBoxHistory = async function (key) {
+    const boxHistory = await this.dbBentoBoxes.createReadStream()
+    let boxData = []
+    for await (const { key, value } of boxHistory) {
+      boxData.push({ key, value })
+    }
+    return boxData
+  }
+
+  /**
+   * delete nxp ref contract from peer library
+   * @method deleteBentoBox
+  */
+  deleteBentoBox = async function (box) {
+    const deleteStatus = await this.dbBentoBoxes.del(box.id)
+    let deleteInfo = {}
+    deleteInfo.id = box.id
+    return deleteInfo
+  }
+
+
   /** MODELS */
   /**
    * save model
@@ -698,6 +749,42 @@ class HyperBee extends EventEmitter {
     return nodeData
   }
 
+ //** */ public library **//
+
+  /**
+   * save pair in keystore public network library
+   * @method savePubliclibrary
+   *
+  */
+  savePubliclibrary = async function (refContract) {
+    await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
+    // go query the key are return the info. to ensure data save asplanned.
+    let saveCheck = await this.getPublicLibrary(refContract.data.hash)
+    let returnMessage = {}
+    returnMessage.stored = true
+    returnMessage.type = refContract.reftype
+    returnMessage.key = saveCheck.key
+    returnMessage.contract = saveCheck.value
+    return returnMessage
+  }
+  
+
+  /**
+   * save pair in keystore db
+   * @method savePeerLibrary
+   *
+  */
+  savePeerLibrary = async function (refContract) {
+    await this.dbPeerLibrary.put(refContract.data.hash, refContract.data.contract)
+    let saveCheck = await this.getPeerLibrary(refContract.data.hash)
+    let returnMessage = {}
+    returnMessage.stored = true
+    returnMessage.type = refContract.reftype
+    returnMessage.key = saveCheck.key
+    returnMessage.contract = saveCheck.value
+    return returnMessage
+  }
+
   /**
    * lookup specific result UUID
    * @method getPublicLibrary
@@ -776,6 +863,16 @@ class HyperBee extends EventEmitter {
     return moduleData
   }
 
+  /*  COHERENCE LEDGER  Knowledge Bundle Ledger */
+
+  /**
+   * save kbledger entry
+   * @method saveKBLentry
+   *
+  */
+  saveKBLentry = async function (ledgerEntry) {
+    await this.dbKBledger.put(ledgerEntry.data, ledgerEntry.hash)
+  }
 
   /**
    * get all kbl entries
@@ -792,30 +889,15 @@ class HyperBee extends EventEmitter {
   }
 
   /**
-   * lookup specific result UUID
-   * @method peerResultsItem
-   *
-  */
-  peerResultsItem = async function (dataPrint) {
-    const resultsData = this.dbKBledger.get(dataPrint.resultuuid)
-    return resultsData
-  }
-
-  /**
-   * lookup specific result UUID
-   * @method peerResults
+   * lookup coherence ledger per results id
+   * @method peerLedgerProof
    *
   */
-  peerResults = async function () {
-    const nodeData = this.dbKBledger.createReadStream()
-    let resData = []
-    for await (const { key, value } of nodeData) {
-      resData.push({ key, value })
-    }
-    return resData
+  peerLedgerProof = async function (dataPrint) {
+    const ledgerData = await this.dbKBledger.get(dataPrint.resultuuid)
+    return ledgerData
   }
 
-
   /**
    * get stream data for keystore db
    * @method getStreamHyperbeeDB

package/src/{drive.js → storage/drive.js}

@@ -13,7 +13,7 @@ import fs from 'fs'
 import Hyperdrive from 'hyperdrive'
 import b4a from 'b4a'
 import Fileparser from './fileParser.js'
-import SqliteAdapter from './adapters/sqliteDatabase.js'
+import SqliteAdapter from '../adapters/sqliteDatabase.js'
 import csv from 'csv-parser'
 import { DateTime, Interval } from 'luxon'
 import * as chrono from 'chrono-node'
@@ -54,12 +54,11 @@ class HypDrive extends EventEmitter {
     // One for a file metadata Hyperbee, and one for a content Hypercore
     this.drive = new Hyperdrive(this.core)
     await this.drive.ready()
-
     let startDrivePubkey = {}
     startDrivePubkey.type = 'account'
     startDrivePubkey.action = 'drive-pubkey'
     startDrivePubkey.data =  b4a.toString(this.drive.key, 'hex')
-    this.wsocket.send(JSON.stringify(startDrivePubkey))
+    // this.wsocket.send(JSON.stringify(startDrivePubkey))
   }
 
 

package/src/{fileParser.js → storage/fileParser.js}

@@ -18,7 +18,7 @@ import events from 'events'
 import axios from 'axios'
 import csv from 'csv-parser'
 import crypto from 'crypto'
-import TimeConvert from './adapters/timeConvertor.js'
+import TimeConvert from '../adapters/timeConvertor.js'
 import { DateTime, Interval } from 'luxon'
 import * as chrono from 'chrono-node'
 
@@ -162,6 +162,8 @@ FileParser.prototype.TEMPwebJSONparse = function (fjData) {
 *
 */
 FileParser.prototype.TEMPwebCSVparse = function (fData) {
+  console.log('tem csv file data')
+  console.log(fData[0])
   // match name row number
   let lcounter = 0
   let match = ''
@@ -178,10 +180,14 @@ FileParser.prototype.TEMPwebCSVparse = function (fData) {
       // pairData.timestamp = splitRow[2]
       // pairData.price = splitRow[5]
       // console.log(pairData)
+      console.log('looop')
+      console.log(splitRow)
       if (splitRow[1] !== undefined) {
         extractCol.push(splitRow[fData.context.id])
         // assume data column for now and parse to mills seconds
         let testCH1 = chrono.parseDate(splitRow[fData.context.timestamp])
+        console.log('pare chonon date')
+        console.log(testCH1)
         let parseDate = this.convertTime.testDataExtact(testCH1)
         // let parseDate = DateTime.fromISO(splitRow[0])
         // let millDate = parseDate.toMillis()

package/test/datacommands/files/data/jan3-bitcoin.csv

@@ -0,0 +1,17 @@
+datetime,tz_cd,price,volume,source,site_no
+2009-01-03 01:30,PST,0,0,bitcoin,0
+2010-01-03 01:30,PST,0,0,bitcoin,0
+2011-01-03 01:30,PST,0.3,14300,bitcoin,0
+2012-01-03 01:30,PST,4.9,125170,bitcoin,0
+2013-01-03 01:30,PST,13.4,18030,bitcoin,0
+2014-01-03 01:30,PST,884.3,9710,bitcoin,0
+2015-01-03 01:30,PST,287.1,47370,bitcoin,0
+2016-01-03 01:30,PST,430.7,54830,bitcoin,0
+2017-01-03 01:30,PST,1033.3,547900,bitcoin,0
+2018-01-03 01:31,PST,15156.6,106540,bitcoin,0
+2019-01-03 01:32,PST,3780.1,468450,bitcoin,0
+2020-01-03 01:33,PST,7343.1,938290,bitcoin,0
+2021-01-03 01:34,PST,32958.9,155210,bitcoin,0
+2022-01-03 01:35,PST,46430.2,41060,bitcoin,0
+2023-01-03 01:36,PST,16674.2,178730,bitcoin,0
+2024-01-03 01:37,PST,42836.1,117650,bitcoin,0

package/test/datacommands/files/large-csv.test.js

@@ -0,0 +1,7 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+
+describe('Large CSV File Operations Tests', () => {
+  it.skip('Large CSV file operations', () => {
+    // This test is skipped for now
+  })
+})

package/test/datacommands/files/small-csv.test.js

@@ -0,0 +1,76 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import Corestore from 'corestore'
+import fs from 'fs'
+import path from 'path'
+import os from 'os'
+import Hyperswarm from 'hyperswarm'
+import goodbye from 'graceful-goodbye'
+import b4a from 'b4a'
+import DriveWorker from '../../../src/storage/drive.js'
+
+// Get the path to the test data directory
+const testDataPath = path.join(__dirname, 'data')
+const testCSVPath = path.join(testDataPath, 'jan3-bitcoin.csv')
+
+describe('CSV File Operations Tests', () => {
+  let store
+  let swarm
+  let clientDrive
+
+  beforeEach(async () => {
+
+    store = new Corestore(os.homedir() + '/.test-file-store')
+    swarm = new Hyperswarm()
+    // make replication possible
+    swarm.on('connection', conn => store.replicate(conn))
+    goodbye(() => swarm.destroy())
+
+    // Create HypDrive instance
+    clientDrive = new DriveWorker(store, swarm)
+
+    // Initialize drive
+    await clientDrive.setupHyperdrive()
+  })
+
+  afterEach(async () => {
+    // Clean up
+  })
+
+  it('should save and retrieve small CSV file', async () => {
+    // 1. Read the actual CSV file
+    const csvContent = fs.readFileSync(testCSVPath, 'utf-8')
+    const csvFileName = path.basename(testCSVPath)
+
+    // 2. Save CSV file
+    await clientDrive.hyperdriveFilesave('text/csv', csvFileName, csvContent)
+
+    // 3. Verify file is saved
+    const clientFileContent = await clientDrive.drive.get('text/csv/' + csvFileName)
+    expect(clientFileContent.toString()).toBe(csvContent)
+
+    // 4. Verify file metadata
+    const fileInfo = await clientDrive.drive.entry('text/csv/' + csvFileName)
+    expect(fileInfo).toBeDefined()
+    expect(fileInfo.value.blob).toBeDefined()
+    expect(fileInfo.value.blob.byteLength).toBe(csvContent.length)
+
+    // 5. Verify file structure (headers)
+    const fileLines = csvContent.split('\n')
+    const headers = fileLines[0].split(',')
+    expect(headers.length).toBeGreaterThan(0)
+    expect(headers[0]).toBeDefined()
+
+    // 6. Verify specific 2015 entry
+    const entry2015 = fileLines[7] // 7th line (0-indexed)
+    expect(entry2015).toBe('2015-01-03 01:30,PST,287.1,47370,bitcoin,0')
+
+    // 7. Verify specific values in 2015 entry
+    const entryValues = entry2015.split(',')
+    expect(entryValues[0]).toBe('2015-01-03 01:30')
+    expect(entryValues[1]).toBe('PST')
+    expect(entryValues[2]).toBe('287.1')
+    expect(entryValues[3]).toBe('47370')
+    expect(entryValues[4]).toBe('bitcoin')
+    expect(entryValues[5]).toBe('0')
+  })
+})

package/test/datacommands/ledger/save-get-ledger.test.js

@@ -0,0 +1,117 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest'
+import Corestore from 'corestore'
+import Hyperswarm from 'hyperswarm'
+import Hyperbee from 'hyperbee'
+import b4a from 'b4a'
+import hashObject from 'object-hash'
+import path from 'path'
+import os from 'os'
+import goodbye from 'graceful-goodbye'
+import BeeWorker from '../../../src/storage/bees.js'
+
+// Get the path to the test data directory
+const testDataPath = path.join(__dirname, 'data')
+
+describe('Hyperbee Ledger Tests', () => {
+  let store
+  let swarm
+  let bee
+
+  beforeAll(async () => {
+    console.log('start of before each')
+    store = new Corestore(os.homedir() + '/.test-ledger-store')
+    swarm = new Hyperswarm()
+    // make replication possible
+    swarm.on('connection', conn => store.replicate(conn))
+    goodbye(() => swarm.destroy())
+
+    // Additional wait for full initialization
+    await new Promise((resolve) => setTimeout(resolve, 4000))
+    console.log('wait afoter')
+    // Create BeeWorker instance
+    bee = new BeeWorker(store, swarm)
+    // await bee.setupHyperbee()  vitest have issues starting so many core at once
+    const core5 = store.get({ name: 'kbledger' })
+    bee.dbKBledger = new Hyperbee(core5, {
+      keyEncoding: 'utf-8',
+      valueEncoding: 'json'
+    })
+    await bee.dbKBledger.ready()
+    await new Promise((resolve) => setTimeout(resolve, 2000))
+  })
+
+  afterAll(async () => {
+    console.log('Cleaning up test environment')
+    // await swarm.destroy()
+    await bee.dbKBledger.close()
+    bee = null
+    store = null
+    // swarm = null
+  }, 20000) // 20 second timeout for cleanup
+
+  it('should perform basic Hyperbee operations', async () => {
+    // Test data
+    console.log('start of test')
+    const testHash = 'test-hash'
+    const testData = { message: 'Hello Hyperbee!' }
+
+    // 1. Put data
+    await bee.dbKBledger.put(testHash, testData)
+    
+    // 2. Get data
+    const result = await bee.dbKBledger.get(testHash)
+    expect(result.value).toEqual(testData)
+
+    // 3. Batch operations with 32-byte keys
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const batch = bee.dbKBledger.batch()
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+    await batch.put(key1, res1)
+    await batch.put(key2, res2)
+    await batch.flush()
+
+    // 4. CreateReadStream
+    const stream = bee.dbKBledger.createReadStream()
+    const entries = []
+    for await (const entry of stream) {
+      entries.push(entry)
+    }
+    expect(entries.length).toBeGreaterThan(0)
+
+    // 5. Check version
+    expect(bee.dbKBledger.version).toBeGreaterThan(0)
+
+    // 6. Verify batch operations
+    const result1 = await bee.dbKBledger.get(key1)
+    expect(result1.key).toEqual(key1)
+    const result2 = await bee.dbKBledger.get(key2)
+    expect(result2.key).toEqual(key2)
+  })
+
+  it('should handle key value operations', async () => {
+    // Test data
+    console.log('start of test2')
+    let res1 = { bbid: '1212121212', data: { value: [1, 2, 3] }}
+    let res2 = { bbid: '3232323232', data: { value: [4, 5, 6] }}
+    const key1 = hashObject(res1)
+    const key2 = hashObject(res2)
+
+    // Put multiple values
+    await bee.dbKBledger.put(key1, res1)
+    await bee.dbKBledger.put(key2, res2)
+
+    // Get values
+    const result1 = await bee.dbKBledger.get(key1)
+    const result2 = await bee.dbKBledger.get(key2)
+    expect(result1.value.data.value[0]).toEqual(1)
+    expect(result2.value.data.value[0]).toEqual(4)
+
+    // Delete value
+    await bee.dbKBledger.del(key1)
+    const deletedResult = await bee.dbKBledger.get(key1)
+    expect(deletedResult).toBeNull()
+    console.log('end of test2')
+  })
+})