holepunch-hop 0.2.6 → 0.2.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "holepunch-hop",
-  "version": "0.2.6",
+  "version": "0.2.8",
   "description": "data interface to HOP",
   "main": "src/index.js",
   "type": "module",
@@ -38,7 +38,8 @@
     "hyperswarm": "^4.3.5",
     "localdrive": "github:holepunchto/localdrive",
     "luxon": "^3.4.4",
-    "chrono-node": "^2.7.5"
+    "chrono-node": "^2.7.5",
+    "sqlite3": "^5.1.7"
   },
   "devDependencies": {
     "assert": "^2.0.0",

package/src/adapters/sqliteDatabase.js

@@ -0,0 +1,163 @@
+'use strict'
+/**
+*  Adapter utility for sqlite database
+*
+*
+* @class SqliteAdapter
+* @package    holepunch-hop
+* @copyright  Copyright (c) 2024 James Littlejohn
+* @license    http://www.gnu.org/licenses/old-licenses/gpl-3.0.html
+* @version    $Id$
+*/
+import EventEmitter from 'events'
+import sqlite3 from 'sqlite3'
+
+class SqliteAdapter extends EventEmitter {
+
+  constructor(options) {
+    super()
+    this.dataBase = {}
+  }
+
+  /**
+  * setup new sqlite Database
+  * @method newDatabase
+  *
+  */
+  newDatabase = async function (db) {
+    let newSqlite = {}
+    this.createDbConnection(db)
+     // ask for tables 
+    let tables = await this.discoverTables()
+    // let header = await this.discoverColumns()
+    newSqlite.tables = tables
+    // newSqlite.headers = header
+    return newSqlite
+  }
+
+  /**
+  * discover colum names in a table
+  * @method tableQuery
+  *
+  */
+  tableQuery = async function (data) {
+    let newSqlite = {}
+    this.createDbConnection(data.file)
+     // ask for tables 
+    let headers = await this.discoverColumns(data.table)
+    newSqlite.headers = headers
+    return newSqlite
+  }
+
+  /**
+  * setup sqlite db and get tables and columns
+  * @method createDbConnection
+  *
+  */
+  createDbConnection = function (db) {
+    this.dataBase = new sqlite3.Database(db, (error) => {
+      if (error) {
+        return console.error(error.message)
+      }
+    })
+    console.log("Connection with SQLite has been established")
+  }
+
+
+  /**
+  * setup sqlite db and get tables and columns
+  * @method discoverTables
+  *
+  */
+  discoverTables = async function () {
+    // tables available
+    let tables = []
+    const res = await new Promise((resolve, reject) => {
+      let sqlTable = `SELECT name FROM sqlite_master WHERE type='table'`
+
+      this.dataBase.all(sqlTable, [], (err, rows) => {
+      if (err) {
+        reject(err)
+      }
+        rows.forEach((row) => {
+	  tables.push(row)
+        })
+       resolve(tables)
+      })
+    })
+    return res
+  }
+
+  /**
+  * setup sqlite db and get tables and columns
+  * @method discoverColumns
+  *
+  */
+  discoverColumns = async function (table) {  
+    // columns in table
+    let header = []
+    const res = await new Promise((resolve, reject) => {
+	    let sqlTableCols = `PRAGMA table_info('` + table + `')`
+	    this.dataBase.all(sqlTableCols, [], (err, rows) => {
+	      if (err) {
+		reject(err)
+	      }
+	      rows.forEach((row) => {
+		      header.push(row)
+	      })
+	      resolve(header)
+	    })
+	})
+    return res
+  }
+
+  /**
+  * query a table for data
+  * @method queryTable
+  *
+  */
+  queryTable = async function (table) {
+    // columns in table
+    let data = []
+    const res = await new Promise((resolve, reject) => {
+	    let sqlQuery = `SELECT * FROM MI_BAND_ACTIVITY_SAMPLE WHERE DEVICE_ID = 3 ORDER BY TIMESTAMP DESC LIMIT 1400` // AND TIMESTAMP BETWEEN 1627677840 AND 1627678380`
+
+	    this.dataBase.all(sqlQuery, [], (err, rows) => {
+	      if (err) {
+		reject(err)
+	      }
+	      rows.forEach((row) => {
+		data.push(row)
+	      })
+	      resolve(data)
+	    })
+	})
+    return res
+  }
+
+  /**
+  * query device table
+  * @method deviceQuery
+  *
+  */
+  deviceQuery = async function (table) {
+    let data = []
+    const res = await new Promise((resolve, reject) => {
+	    let sqlQuery = `SELECT * FROM ` + table
+
+	    this.dataBase.all(sqlQuery, [], (err, rows) => {
+	      if (err) {
+		reject(err)
+	      }
+	      rows.forEach((row) => {
+		data.push(row)
+	      })
+	      resolve(data)
+	    })
+	})
+    return res
+  }
+
+}
+
+export default SqliteAdapter

package/src/adapters/timeConvertor.js

@@ -0,0 +1,43 @@
+'use strict'
+/**
+*  standard time inputs
+*
+*
+* @class TimeConvertor
+* @package    holepunch-hop
+* @copyright  Copyright (c) 2024 James Littlejohn
+* @license    http://www.gnu.org/licenses/old-licenses/gpl-3.0.html
+* @version    $Id$
+*/
+import EventEmitter from 'events'
+import { DateTime, Interval } from 'luxon'
+import * as chrono from 'chrono-node'
+
+class TimeConvertor extends EventEmitter {
+
+  constructor(options) {
+    super()
+  }
+
+  /**
+  * try to match to known time formats
+  * @method testDataExtact
+  *
+  */
+  testDataExtact = function (sampleDate) {
+    let parseDate0 = DateTime.fromISO(sampleDate)
+    let parseDate1 = DateTime.fromHTTP(sampleDate)
+    let parseDate2 = DateTime.fromJSDate(sampleDate)
+    /* let parseDate3 = DateTime.fromFormat(sampleDate, "dd-MM-yyyy HH:MM")
+    console.log(parseDate3)
+    let parseDate4 = DateTime.local(sampleDate)
+    console.log(parseDate4)
+    // .fromJSDate(sampleDate) // .fromHTTP(sampleDate) //  fromFormat(sampleDate, "YYY-MM-DD ")  //.fromISO(sampleDate) // or DateTime. fromFormat("23-06-2019", "dd-MM-yyyy") .(splitRow[0])//  new Date(splitRow[0])
+    // console.log(parseDate) */
+    let millDate = parseDate2.toMillis()
+    return millDate
+  }
+
+}
+
+export default TimeConvertor

package/src/bees.js

@@ -113,6 +113,7 @@ class HyperBee extends EventEmitter {
    *
   */
   savePubliclibrary = async function (refContract) {
+    console.log(refContract)
     let beeSave = await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
     // go query the key are return the info. to ensure data save asplanned.
     let saveCheck = await this.getPublicLibrary(refContract.data.hash)
@@ -131,8 +132,8 @@ class HyperBee extends EventEmitter {
    *
   */
   savePeerLibrary = async function (refContract) {
-    await this.dbPeerLibrary.put(refContract.hash, refContract.contract)
-    let saveCheck = await this.getPeerLibrary(refContract.hash)
+    await this.dbPeerLibrary.put(refContract.data.hash, refContract.data.contract)
+    let saveCheck = await this.getPeerLibrary(refContract.data.hash)
     let returnMessage = {}
     returnMessage.stored = true
     returnMessage.type = refContract.reftype
@@ -398,6 +399,21 @@ class HyperBee extends EventEmitter {
 
   }
 
+
+
+  /**
+   * delete nxp ref contract public
+   * @method deleteRefcontPubliclibrary
+   *
+  */
+  deleteRefcontPubliclibrary = async function (nxpID) {
+    let deleteInfo = {}
+    let deleteStatus = await this.dbPublicLibrary.del(nxpID)
+    deleteInfo.success = deleteStatus
+    deleteInfo.nxp = nxpID
+    return deleteInfo
+  }
+
   /**
    * delete nxp ref contract from peer library
    * @method deleteRefcontPeerlibrary

package/src/drive.js

@@ -13,7 +13,10 @@ import fs from 'fs'
 import Hyperdrive from 'hyperdrive'
 import b4a from 'b4a'
 import Fileparser from './fileParser.js'
+import SqliteAdapter from './adapters/sqliteDatabase.js'
 import csv from 'csv-parser'
+import { DateTime, Interval } from 'luxon'
+import * as chrono from 'chrono-node'
 
 class HypDrive extends EventEmitter {
 
@@ -24,6 +27,8 @@ class HypDrive extends EventEmitter {
     this.swarm = swarm
     this.drive = {}
     this.fileUtility = new Fileparser('')
+    this.AdapterSqlite = new SqliteAdapter()
+    this.dataBase = {}
     // this.setupHyperdrive()
   }
 
@@ -83,7 +88,6 @@ class HypDrive extends EventEmitter {
   */
   listFilesFolder = function (folder) {
     const stream = this.drive.list('') //  [options])
-    // console.log(stream)
     // Handle stream events --> data, end, and error
     let dataDrive = []
     stream.on('data', function(chunk) {
@@ -150,6 +154,37 @@ class HypDrive extends EventEmitter {
   }
 
 
+  /**
+   * save csv data to hyperdrive file
+   * @method saveSqliteFirst 
+   *
+  */
+  saveSqliteFirst = async function (path, name, data) {
+    let fileResponse = {}
+    // file input management
+    // protocol to save original file
+    let newPathFile = await this.hyperdriveFilesave(path, name, data)
+    // extract table and then table columns
+    // extract out the headers name for columns
+    const parseData = await this.SQLiteSetup(name)
+    fileResponse.filename = name
+    fileResponse.header = parseData.headers
+    fileResponse.tables = parseData.tables
+    return fileResponse
+  }
+
+
+  /**
+   * blind sqlite lookup of data (maybe restricture no. of rows of data iniitally)
+   * @method blindDataSqlite 
+   *
+  */
+  blindDataSqlite = async function (dataInfo) {
+     let parseData = await this.SQLiteQuery(dataInfo)
+     return parseData
+  }
+
+
   /**
    * save to hyperdrive file
    * @method hyperdriveFilesave 
@@ -184,8 +219,6 @@ class HypDrive extends EventEmitter {
         }
       }))
     }
-
-
     return hyperdrivePath
   }
 
@@ -206,20 +239,108 @@ class HypDrive extends EventEmitter {
    * rebuidl file and give directory location
    * @method hyperdriveLocalfile
    *
-   */
+  */
   hyperdriveLocalfile = async function (path) {
     // File reads to buffer and recreate file
     // const bufFromGet2 = await this.drive.get(path)
     const { value: entry } = await this.drive.entry(path)
     const blobs = await this.drive.getBlobs()
     const bufFromEntry = await blobs.get(entry.blob)
-
     let localFile = 'localdb'
-    // fs.writeFileSync(localFile, bufFromGet2)
     fs.writeFileSync(localFile, bufFromEntry)
     return localFile
   }
 
+  /**
+  *  set file path, read and make sqlite3 connect db
+  * @method SQLiteSetup
+  *
+  */
+  SQLiteSetup = async function (file) {
+    // const stream = this.liveDataAPI.DriveFiles.listFilesFolder('sqlite/')
+    let dbFile = await this.hyperdriveLocalfile('sqlite/' + file)
+    let summarySQLinfo = await this.AdapterSqlite.newDatabase(dbFile)
+    return summarySQLinfo
+  }
+  
+  /**
+  *  set file path, read and make sqlite3 connect db
+  * @method SQLiteSourceSetup
+  *
+  */
+  SQLiteSourceSetup = async function (data) {
+    // const stream = this.liveDataAPI.DriveFiles.listFilesFolder('sqlite/')
+    let dbFile = await this.hyperdriveLocalfile('sqlite/' + data.db)
+    data.file = dbFile
+    let summarySQLinfo = await this.AdapterSqlite.tableQuery(data)
+    return summarySQLinfo
+  }
+
+
+  /**
+  *  ask for device tables and info.
+  * @method SQLiteDeviceSetup
+  *
+  */
+  SQLiteDeviceSetup = async function (data) {
+    // const stream = this.liveDataAPI.DriveFiles.listFilesFolder('sqlite/')
+    let dbFile = await this.hyperdriveLocalfile('sqlite/' + data.db)
+    data.file = dbFile
+    let summaryTable = await this.AdapterSqlite.tableQuery(data)
+    let summaryDevice = await this.AdapterSqlite.deviceQuery(data.table)
+    let summarySQLinfo = {}
+    summarySQLinfo.tables = summaryTable
+    summarySQLinfo.devices = summaryDevice
+    return summarySQLinfo
+  }
+
+  /**
+  *  set file path, read and make sqlite3 connect db
+  * @method SQLiteQuery
+  *
+  */
+  SQLiteQuery = async function (dataInfo) {
+     let timestampCol = ''
+    // is the sqliite database sill accive?
+    // const stream = this.liveDataAPI.DriveFiles.listFilesFolder('sqlite/')
+    let dbFile = await this.hyperdriveLocalfile('sqlite/' + dataInfo.file.file)
+    let queryData = await this.AdapterSqlite.queryTable(dataInfo)
+    let contextKeys = Object.keys(queryData[0])
+    timestampCol = contextKeys[dataInfo.context.timestamp]
+    // now prepare into data and labels
+    let blindData = {}
+    let extractCol = []
+    let extractLabel = []
+    for (let rowi of queryData) {
+	extractCol.push(rowi[dataInfo.context.name.name])
+	// assume data column for now and parse to mills seconds
+	let testCH1 = chrono.parseDate(rowi[timestampCol])
+	let parseDate = testCH1 * 1000 // this.testDataExtact(testCH1)
+	extractLabel.push(parseDate)
+    }
+    blindData.data = extractCol
+    blindData.label = extractLabel
+    return blindData
+  }
+
+
+	/**
+	* try to match to known time formats
+	* @method testDataExtact
+	*
+	*/
+	testDataExtact = function (sampleDate) {
+	  let parseDate0 = DateTime.fromISO(sampleDate)
+	  let parseDate1 = DateTime.fromHTTP(sampleDate)
+	  let parseDate2 = DateTime.fromJSDate(sampleDate)
+	  /* let parseDate3 = DateTime.fromFormat(sampleDate, "dd-MM-yyyy HH:MM")
+	  let parseDate4 = DateTime.local(sampleDate)
+	  // .fromJSDate(sampleDate) // .fromHTTP(sampleDate) //  fromFormat(sampleDate, "YYY-MM-DD ")  //.fromISO(sampleDate) // or DateTime. fromFormat("23-06-2019", "dd-MM-yyyy") .(splitRow[0])//  new Date(splitRow[0])
+	   */
+	  let millDate = parseDate2.toMillis()
+	  return millDate
+	}
+
   /**
   *  taken in csv file and read per line
   * @method readCSVfile

package/src/fileParser.js

@@ -11,7 +11,6 @@ import atob from 'atob'
 * @license    http://www.gnu.org/licenses/old-licenses/gpl-3.0.html
 * @version    $Id$
 */
-
 import fs from 'fs'
 import os from 'os'
 import util from 'util'
@@ -19,12 +18,14 @@ import events from 'events'
 import axios from 'axios'
 import csv from 'csv-parser'
 import crypto from 'crypto'
+import TimeConvert from './adapters/timeConvertor.js'
 import { DateTime, Interval } from 'luxon'
 import * as chrono from 'chrono-node'
 
 var FileParser = function (path) {
   events.EventEmitter.call(this)
   this.storepath = path
+  this.convertTime = new TimeConvert()
 }
 
 /**
@@ -162,7 +163,7 @@ FileParser.prototype.TEMPwebCSVparse = function (fData) {
         extractCol.push(splitRow[fData.context.id])
         // assume data column for now and parse to mills seconds
         let testCH1 = chrono.parseDate(splitRow[fData.context.timestamp])
-        let parseDate = this.testDataExtact(testCH1)
+        let parseDate = this.convertTime.testDataExtact(testCH1)
         // let parseDate = DateTime.fromISO(splitRow[0])
         // let millDate = parseDate.toMillis()
         extractLabel.push(parseDate)
@@ -173,30 +174,9 @@ FileParser.prototype.TEMPwebCSVparse = function (fData) {
   let extractedPair = {}
   extractedPair.label = extractLabel
   extractedPair.data = extractCol
-  console.log('finsiehd')
-  console.log(extractedPair)
   return extractedPair
 }
 
-/**
-* try to match to known time formats
-* @method testDataExtact
-*
-*/
-FileParser.prototype.testDataExtact = function (sampleDate) {
-  let parseDate0 = DateTime.fromISO(sampleDate)
-  let parseDate1 = DateTime.fromHTTP(sampleDate)
-  let parseDate2 = DateTime.fromJSDate(sampleDate)
-  /* let parseDate3 = DateTime.fromFormat(sampleDate, "dd-MM-yyyy HH:MM")
-  console.log(parseDate3)
-  let parseDate4 = DateTime.local(sampleDate)
-  console.log(parseDate4)
-  // .fromJSDate(sampleDate) // .fromHTTP(sampleDate) //  fromFormat(sampleDate, "YYY-MM-DD ")  //.fromISO(sampleDate) // or DateTime. fromFormat("23-06-2019", "dd-MM-yyyy") .(splitRow[0])//  new Date(splitRow[0])
-  // console.log(parseDate) */
-  let millDate = parseDate2.toMillis()
-  return millDate
-}
-
 /**
 * read csv headers and extract info
 * @method extractCSVHeaderInfo