holepunch-hop 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/bees.js +322 -8
- package/src/drive.js +123 -15
- package/src/fileParser.js +0 -5
- package/src/index.js +53 -5
- package/src/peers.js +25 -0
- package/test/large-csv.test.js +9 -0
package/package.json
CHANGED
package/src/bees.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Manage HyperBee key store datastore
|
|
4
4
|
*
|
|
5
|
-
* @class
|
|
6
|
-
* @package
|
|
5
|
+
* @class HyperBee
|
|
6
|
+
* @package HyperBee
|
|
7
7
|
* @copyright Copyright (c) 2024 James Littlejohn
|
|
8
8
|
* @license http://www.gnu.org/licenses/old-licenses/gpl-3.0.html
|
|
9
9
|
* @version $Id$
|
|
@@ -96,6 +96,66 @@ class HyperBee extends EventEmitter {
|
|
|
96
96
|
await this.dbKBledger.ready()
|
|
97
97
|
// this.client.replicate(this.dbKBledger.feed)
|
|
98
98
|
beePubkeys.push({store:'kbledger', pubkey: b4a.toString(core5.key, 'hex')})
|
|
99
|
+
// stores of cues, media, research, markers, products/treatments
|
|
100
|
+
|
|
101
|
+
const core7 = this.store.get({ name: 'bentocues' })
|
|
102
|
+
this.dbBentocues = new Hyperbee(core7, {
|
|
103
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
104
|
+
valueEncoding: 'json' // same options as above
|
|
105
|
+
})
|
|
106
|
+
await this.dbBentocues.ready()
|
|
107
|
+
beePubkeys.push({store:'bentocues', pubkey: b4a.toString(core7.key, 'hex')})
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
const core8 = this.store.get({ name: 'bentodecisions' })
|
|
111
|
+
this.dbBentodecisions = new Hyperbee(core8, {
|
|
112
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
113
|
+
valueEncoding: 'json' // same options as above
|
|
114
|
+
})
|
|
115
|
+
await this.dbBentodecisions.ready()
|
|
116
|
+
beePubkeys.push({store:'bentodecisions', pubkey: b4a.toString(core8.key, 'hex')})
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
const core9 = this.store.get({ name: 'bentomarkers' })
|
|
120
|
+
this.dbBentomarkers = new Hyperbee(core9, {
|
|
121
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
122
|
+
valueEncoding: 'json' // same options as above
|
|
123
|
+
})
|
|
124
|
+
await this.dbBentomarkers.ready()
|
|
125
|
+
beePubkeys.push({store:'bentomarkers', pubkey: b4a.toString(core9.key, 'hex')})
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
const core10 = this.store.get({ name: 'research' })
|
|
129
|
+
this.dbBentoresearch = new Hyperbee(core10, {
|
|
130
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
131
|
+
valueEncoding: 'json' // same options as above
|
|
132
|
+
})
|
|
133
|
+
await this.dbBentoresearch.ready()
|
|
134
|
+
beePubkeys.push({store:'research', pubkey: b4a.toString(core10.key, 'hex')})
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
const core11 = this.store.get({ name: 'bentoproducts' })
|
|
138
|
+
this.dbBentoproducts = new Hyperbee(core11, {
|
|
139
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
140
|
+
valueEncoding: 'json' // same options as above
|
|
141
|
+
})
|
|
142
|
+
await this.dbBentoproducts.ready()
|
|
143
|
+
beePubkeys.push({store:'bentoproducts', pubkey: b4a.toString(core11.key, 'hex')})
|
|
144
|
+
|
|
145
|
+
const core12 = this.store.get({ name: 'bentomedia' })
|
|
146
|
+
this.dbBentomedia = new Hyperbee(core12, {
|
|
147
|
+
keyEncoding: 'utf-8', // can be set to undefined (binary), utf-8, ascii or and abstract-encoding
|
|
148
|
+
valueEncoding: 'json' // same options as above
|
|
149
|
+
})
|
|
150
|
+
await this.dbBentomedia.ready()
|
|
151
|
+
beePubkeys.push({store:'bentomedia', pubkey: b4a.toString(core12.key, 'hex')})
|
|
152
|
+
|
|
153
|
+
// await this.deleteBentocue({ cueid: '7da21b8d50c72f94595fd9617bfa2609c90d5d9c'})
|
|
154
|
+
// await this.deleteBentocue({ cueid: '8220c425a534634a44a69b6f56babaf26edf2a01'})
|
|
155
|
+
// await this.deleteBentocue({ cueid: 'ec4890f6c0b0d8f4a313136b4e7c9f425b1977a5'})
|
|
156
|
+
// await this.deleteBentocue({ cueid: 'eea2825cc91a1093594dffa9a953cbce3f77860c'})
|
|
157
|
+
// await this.deleteBentocue({ cueid: 'eee097d964fae67a6dd39abc02b0e9fd7c5f71bc'})
|
|
158
|
+
|
|
99
159
|
this.emit('hbee-live')
|
|
100
160
|
// return beePubkeys
|
|
101
161
|
let startBeePubkey = {}
|
|
@@ -112,6 +172,8 @@ class HyperBee extends EventEmitter {
|
|
|
112
172
|
*
|
|
113
173
|
*/
|
|
114
174
|
savePubliclibrary = async function (refContract) {
|
|
175
|
+
console.log('savePubliclibrary')
|
|
176
|
+
console.log(refContract)
|
|
115
177
|
let beeSave = await this.dbPublicLibrary.put(refContract.data.hash, refContract.data.contract)
|
|
116
178
|
// go query the key are return the info. to ensure data save asplanned.
|
|
117
179
|
let saveCheck = await this.getPublicLibrary(refContract.data.hash)
|
|
@@ -212,8 +274,8 @@ class HyperBee extends EventEmitter {
|
|
|
212
274
|
*
|
|
213
275
|
*/
|
|
214
276
|
saveSpaceHistory = async function (spaceContract) {
|
|
215
|
-
await this.dbBentospaces.put(spaceContract.space.
|
|
216
|
-
let checkSave = await this.getBentospace(spaceContract.space.
|
|
277
|
+
await this.dbBentospaces.put(spaceContract.space.cueid, spaceContract)
|
|
278
|
+
let checkSave = await this.getBentospace(spaceContract.space.cueid)
|
|
217
279
|
return checkSave
|
|
218
280
|
}
|
|
219
281
|
|
|
@@ -223,8 +285,8 @@ class HyperBee extends EventEmitter {
|
|
|
223
285
|
*
|
|
224
286
|
*/
|
|
225
287
|
saveBentospace = async function (spaceContract) {
|
|
226
|
-
await this.dbBentospaces.put(spaceContract.
|
|
227
|
-
let checkSave = await this.getBentospace(spaceContract.
|
|
288
|
+
await this.dbBentospaces.put(spaceContract.cueid, spaceContract)
|
|
289
|
+
let checkSave = await this.getBentospace(spaceContract.cueid)
|
|
228
290
|
return checkSave
|
|
229
291
|
}
|
|
230
292
|
|
|
@@ -238,15 +300,267 @@ class HyperBee extends EventEmitter {
|
|
|
238
300
|
return nodeData
|
|
239
301
|
}
|
|
240
302
|
|
|
303
|
+
/**
|
|
304
|
+
* lookup bentospaces all
|
|
305
|
+
* @method getAllBentospaces
|
|
306
|
+
*
|
|
307
|
+
*/
|
|
308
|
+
getAllBentospaces = async function () {
|
|
309
|
+
const spacesHistory = await this.dbBentospaces.createReadStream()
|
|
310
|
+
let spacesData = []
|
|
311
|
+
for await (const { key, value } of spacesHistory) {
|
|
312
|
+
spacesData.push({ key, value })
|
|
313
|
+
}
|
|
314
|
+
return spacesData
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
|
|
241
318
|
/**
|
|
242
319
|
* delete nxp ref contract from peer library
|
|
243
320
|
* @method deleteBentospace
|
|
244
321
|
*
|
|
245
322
|
*/
|
|
246
323
|
deleteBentospace = async function (space) {
|
|
247
|
-
const deleteStatus = await this.dbBentospaces.del(space.
|
|
324
|
+
const deleteStatus = await this.dbBentospaces.del(space.cueid)
|
|
325
|
+
let deleteInfo = {}
|
|
326
|
+
deleteInfo.spaceid = space.cueid
|
|
327
|
+
return deleteInfo
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** CUES */
|
|
331
|
+
/**
|
|
332
|
+
* save cues
|
|
333
|
+
* @method saveCues
|
|
334
|
+
*
|
|
335
|
+
*/
|
|
336
|
+
saveCues = async function (cuesInfo) {
|
|
337
|
+
await this.dbBentocues.put(cuesInfo.cueid, cuesInfo.data)
|
|
338
|
+
let checkSave = await this.getCues(cuesInfo.cueid)
|
|
339
|
+
return checkSave
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
/**
|
|
343
|
+
* get one cue by id
|
|
344
|
+
* @method getCues
|
|
345
|
+
*
|
|
346
|
+
*/
|
|
347
|
+
getCues = async function (key) {
|
|
348
|
+
const nodeData = await this.dbBentocues.get(key)
|
|
349
|
+
return nodeData
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/**
|
|
353
|
+
* get all cuees
|
|
354
|
+
* @method getCuesHistory
|
|
355
|
+
*
|
|
356
|
+
*/
|
|
357
|
+
getCuesHistory = async function (key) {
|
|
358
|
+
const cuesHistory = await this.dbBentocues.createReadStream()
|
|
359
|
+
let cuesData = []
|
|
360
|
+
for await (const { key, value } of cuesHistory) {
|
|
361
|
+
cuesData.push({ key, value })
|
|
362
|
+
}
|
|
363
|
+
return cuesData
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/**
|
|
367
|
+
* delete nxp ref contract from peer library
|
|
368
|
+
* @method deleteBentocue
|
|
369
|
+
*/
|
|
370
|
+
deleteBentocue = async function (cue) {
|
|
371
|
+
const deleteStatus = await this.dbBentocues.del(cue.cueid)
|
|
372
|
+
let deleteInfo = {}
|
|
373
|
+
deleteInfo.spaceid = cue.cueid
|
|
374
|
+
return deleteInfo
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
/** MEDIA */
|
|
378
|
+
/**
|
|
379
|
+
* save media
|
|
380
|
+
* @method saveMedia
|
|
381
|
+
*
|
|
382
|
+
*/
|
|
383
|
+
saveMedia = async function (mediaInfo) {
|
|
384
|
+
await this.dbBentomedia.put(mediaInfo.cueid, mediaInfo.data)
|
|
385
|
+
let checkSave = await this.getMedia(mediaInfo.cueid)
|
|
386
|
+
return checkSave
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* get one cue by id
|
|
391
|
+
* @method getMedia
|
|
392
|
+
*
|
|
393
|
+
*/
|
|
394
|
+
getMedia = async function (key) {
|
|
395
|
+
const nodeData = await this.dbBentomedia.get(key)
|
|
396
|
+
return nodeData
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* get all media
|
|
401
|
+
* @method getMediaHistory
|
|
402
|
+
*
|
|
403
|
+
*/
|
|
404
|
+
getMediaHistory = async function (key) {
|
|
405
|
+
const cuesHistory = await this.dbBentomedia.createReadStream()
|
|
406
|
+
let cuesData = []
|
|
407
|
+
for await (const { key, value } of cuesHistory) {
|
|
408
|
+
cuesData.push({ key, value })
|
|
409
|
+
}
|
|
410
|
+
return cuesData
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* delete nxp ref contract from peer library
|
|
416
|
+
* @method deleteBentomedia
|
|
417
|
+
*/
|
|
418
|
+
deleteBentomedia = async function (media) {
|
|
419
|
+
console.log(media)
|
|
420
|
+
const deleteStatus = await this.dbBentomedia.del(media.id)
|
|
421
|
+
let deleteInfo = {}
|
|
422
|
+
deleteInfo.spaceid = media.id
|
|
423
|
+
return deleteInfo
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
/** RESEARCH */
|
|
427
|
+
/**
|
|
428
|
+
* save research
|
|
429
|
+
* @method saveResearch
|
|
430
|
+
*
|
|
431
|
+
*/
|
|
432
|
+
saveResearch = async function (cuesInfo) {
|
|
433
|
+
await this.dbBentoresearch.put(cuesInfo.cueid, cuesInfo.data)
|
|
434
|
+
let checkSave = await this.getResearch(cuesInfo.cueid)
|
|
435
|
+
return checkSave
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
/**
|
|
439
|
+
* get one cue by id
|
|
440
|
+
* @method getResearch
|
|
441
|
+
*
|
|
442
|
+
*/
|
|
443
|
+
getResearch = async function (key) {
|
|
444
|
+
const nodeData = await this.dbBentoresearch.get(key)
|
|
445
|
+
return nodeData
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/**
|
|
449
|
+
* get all research
|
|
450
|
+
* @method getResearchHistory
|
|
451
|
+
*
|
|
452
|
+
*/
|
|
453
|
+
getResearchHistory = async function (key) {
|
|
454
|
+
const cuesHistory = await this.dbBentoresearch.createReadStream()
|
|
455
|
+
let cuesData = []
|
|
456
|
+
for await (const { key, value } of cuesHistory) {
|
|
457
|
+
cuesData.push({ key, value })
|
|
458
|
+
}
|
|
459
|
+
return cuesData
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* delete contract
|
|
464
|
+
* @method deleteBentoResearch
|
|
465
|
+
*/
|
|
466
|
+
deleteBentoResearch = async function (cue) {
|
|
467
|
+
const deleteStatus = await this.dbBentoresearch.del(cue.id)
|
|
468
|
+
let deleteInfo = {}
|
|
469
|
+
deleteInfo.spaceid = cue.id
|
|
470
|
+
return deleteInfo
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/** MARKER */
|
|
474
|
+
/**
|
|
475
|
+
* save marker
|
|
476
|
+
* @method saveMarker
|
|
477
|
+
*
|
|
478
|
+
*/
|
|
479
|
+
saveMarker = async function (cuesInfo) {
|
|
480
|
+
await this.dbBentomarkers.put(cuesInfo.cueid, cuesInfo.data)
|
|
481
|
+
let checkSave = await this.getMarker(cuesInfo.cueid)
|
|
482
|
+
return checkSave
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* get one cue by id
|
|
487
|
+
* @method getMarker
|
|
488
|
+
*
|
|
489
|
+
*/
|
|
490
|
+
getMarker = async function (key) {
|
|
491
|
+
const nodeData = await this.dbBentomarkers.get(key)
|
|
492
|
+
return nodeData
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/**
|
|
496
|
+
* get all research
|
|
497
|
+
* @method getMarkerHistory
|
|
498
|
+
*
|
|
499
|
+
*/
|
|
500
|
+
getMarkerHistory = async function (key) {
|
|
501
|
+
const cuesHistory = await this.dbBentomarkers.createReadStream()
|
|
502
|
+
let cuesData = []
|
|
503
|
+
for await (const { key, value } of cuesHistory) {
|
|
504
|
+
cuesData.push({ key, value })
|
|
505
|
+
}
|
|
506
|
+
return cuesData
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
/**
|
|
510
|
+
* delete contract
|
|
511
|
+
* @method deleteBentoMarker
|
|
512
|
+
*/
|
|
513
|
+
deleteBentoMarker = async function (cue) {
|
|
514
|
+
const deleteStatus = await this.dbBentomarkers.del(cue.id)
|
|
515
|
+
let deleteInfo = {}
|
|
516
|
+
deleteInfo.spaceid = cue.id
|
|
517
|
+
return deleteInfo
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
/** Product */
|
|
521
|
+
/**
|
|
522
|
+
* save product
|
|
523
|
+
* @method saveProduct
|
|
524
|
+
*
|
|
525
|
+
*/
|
|
526
|
+
saveProduct = async function (cuesInfo) {
|
|
527
|
+
await this.dbBentoproducts.put(cuesInfo.cueid, cuesInfo.data)
|
|
528
|
+
let checkSave = await this.getProduct(cuesInfo.cueid)
|
|
529
|
+
return checkSave
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
/**
|
|
533
|
+
* get one cue by id
|
|
534
|
+
* @method getProduct
|
|
535
|
+
*
|
|
536
|
+
*/
|
|
537
|
+
getProduct = async function (key) {
|
|
538
|
+
const nodeData = await this.dbBentoproducts.get(key)
|
|
539
|
+
return nodeData
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
/**
|
|
543
|
+
* get all prodcut
|
|
544
|
+
* @method getProductHistory
|
|
545
|
+
*
|
|
546
|
+
*/
|
|
547
|
+
getProductHistory = async function (key) {
|
|
548
|
+
const cuesHistory = await this.dbBentoproducts.createReadStream()
|
|
549
|
+
let cuesData = []
|
|
550
|
+
for await (const { key, value } of cuesHistory) {
|
|
551
|
+
cuesData.push({ key, value })
|
|
552
|
+
}
|
|
553
|
+
return cuesData
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* delete contract
|
|
558
|
+
* @method deleteBentoProduct
|
|
559
|
+
*/
|
|
560
|
+
deleteBentoProduct = async function (cue) {
|
|
561
|
+
const deleteStatus = await this.dbBentoproducts.del(cue.id)
|
|
248
562
|
let deleteInfo = {}
|
|
249
|
-
deleteInfo.spaceid =
|
|
563
|
+
deleteInfo.spaceid = cue.id
|
|
250
564
|
return deleteInfo
|
|
251
565
|
}
|
|
252
566
|
|
package/src/drive.js
CHANGED
|
@@ -86,18 +86,9 @@ class HypDrive extends EventEmitter {
|
|
|
86
86
|
* @method listFilesFolder
|
|
87
87
|
*
|
|
88
88
|
*/
|
|
89
|
-
listFilesFolder = function (folder) {
|
|
90
|
-
const stream = this.drive.list(
|
|
91
|
-
|
|
92
|
-
let dataDrive = []
|
|
93
|
-
stream.on('data', function(chunk) {
|
|
94
|
-
dataDrive.push(chunk)
|
|
95
|
-
})
|
|
96
|
-
|
|
97
|
-
stream.on('end', function(chunk) {
|
|
98
|
-
// console.log('stream at end')
|
|
99
|
-
// console.log(dataDrive)
|
|
100
|
-
})
|
|
89
|
+
listFilesFolder = async function (folder) {
|
|
90
|
+
const stream = await this.drive.list(folder) // [options])
|
|
91
|
+
return stream
|
|
101
92
|
}
|
|
102
93
|
|
|
103
94
|
/**
|
|
@@ -222,6 +213,109 @@ class HypDrive extends EventEmitter {
|
|
|
222
213
|
return hyperdrivePath
|
|
223
214
|
}
|
|
224
215
|
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* save a stream of file ie. large file
|
|
219
|
+
* @method hyperdriveStreamSave
|
|
220
|
+
*
|
|
221
|
+
*/
|
|
222
|
+
hyperdriveStreamSave = async function (path, data, first) {
|
|
223
|
+
let ws
|
|
224
|
+
if (first === true) {
|
|
225
|
+
// await this.drive.del(path)
|
|
226
|
+
ws = this.drive.createWriteStream(path)
|
|
227
|
+
ws.write(data)
|
|
228
|
+
}
|
|
229
|
+
// use listener
|
|
230
|
+
this.on('stream-update', (data) => {
|
|
231
|
+
ws.write(data)
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
this.on('stream-complete', async (data) => {
|
|
235
|
+
ws.end()
|
|
236
|
+
ws.once('close', () =>
|
|
237
|
+
console.log('stream-close'),
|
|
238
|
+
await this.checkLargeList(path)
|
|
239
|
+
)
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* check read the file if save large file
|
|
246
|
+
* @method checkLargeList
|
|
247
|
+
*
|
|
248
|
+
*/
|
|
249
|
+
checkLargeList = async function (path) {
|
|
250
|
+
let localthis = this
|
|
251
|
+
let folder = 'test'
|
|
252
|
+
let folderList = await this.listFilesFolder(folder)
|
|
253
|
+
let dataDrive = []
|
|
254
|
+
folderList.on('data', function(chunk) {
|
|
255
|
+
dataDrive.push(chunk)
|
|
256
|
+
})
|
|
257
|
+
|
|
258
|
+
folderList.on('end', async function() {
|
|
259
|
+
for (let file of dataDrive) {
|
|
260
|
+
if (file.key === path) {
|
|
261
|
+
await localthis.firstLineLargeCSV(path)
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
})
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* check read the file if save large file
|
|
269
|
+
* @method checkLargeSave
|
|
270
|
+
*
|
|
271
|
+
*/
|
|
272
|
+
checkLargeSave = async function (path) {
|
|
273
|
+
const rs = this.drive.createReadStream(path)
|
|
274
|
+
for await (const chunk of rs) {
|
|
275
|
+
console.log('rs', chunk.toString()) // => <Buffer ..>
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* check read the file if save large file
|
|
281
|
+
* @method firstLineLargeCSV
|
|
282
|
+
*
|
|
283
|
+
*/
|
|
284
|
+
firstLineLargeCSV = async function (path) {
|
|
285
|
+
let readFile = await this.readCSVfileStream(path)
|
|
286
|
+
let makeString = readFile[0].toString()
|
|
287
|
+
let csvFormat = makeString.split(/\r?\n/)
|
|
288
|
+
let headerList = csvFormat[0].split(",");
|
|
289
|
+
let largeMessage = {}
|
|
290
|
+
largeMessage.type = 'library'
|
|
291
|
+
largeMessage.action = 'PUT-stream'
|
|
292
|
+
largeMessage.task = 'PUT-stream'
|
|
293
|
+
largeMessage.save = true
|
|
294
|
+
largeMessage.data = {}
|
|
295
|
+
largeMessage.data.path = path
|
|
296
|
+
largeMessage.data.columns = headerList
|
|
297
|
+
this.emit('largefile-save', largeMessage)
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* stream save update
|
|
302
|
+
* @method streamSavedata
|
|
303
|
+
*
|
|
304
|
+
*/
|
|
305
|
+
streamSavedata = async function (path, data) {
|
|
306
|
+
this.emit('stream-update', data)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* stream save complete
|
|
311
|
+
* @method streamSaveComplete
|
|
312
|
+
*
|
|
313
|
+
*/
|
|
314
|
+
streamSaveComplete = async function (data) {
|
|
315
|
+
this.emit('stream-update', data)
|
|
316
|
+
this.emit('stream-complete', data)
|
|
317
|
+
}
|
|
318
|
+
|
|
225
319
|
/**
|
|
226
320
|
* read file nav to folder
|
|
227
321
|
* @method hyperdriveReadfile
|
|
@@ -300,7 +394,6 @@ class HypDrive extends EventEmitter {
|
|
|
300
394
|
*
|
|
301
395
|
*/
|
|
302
396
|
SQLiteQuery = async function (dataInfo) {
|
|
303
|
-
console.log('HP--DRIVE--sqlite')
|
|
304
397
|
let timestampCol = ''
|
|
305
398
|
// is the sqliite database sill accive?
|
|
306
399
|
// const stream = this.liveDataAPI.DriveFiles.listFilesFolder('sqlite/')
|
|
@@ -324,7 +417,6 @@ class HypDrive extends EventEmitter {
|
|
|
324
417
|
blindData.label = extractLabel
|
|
325
418
|
return blindData
|
|
326
419
|
} else {
|
|
327
|
-
console.log('no data for that query')
|
|
328
420
|
let blindData = {}
|
|
329
421
|
blindData.data = []
|
|
330
422
|
blindData.label = []
|
|
@@ -362,7 +454,7 @@ class HypDrive extends EventEmitter {
|
|
|
362
454
|
const rs = this.drive.createReadStream(fpath) // 'text/csv/testshed11530500.csv') // '/blob.txt')
|
|
363
455
|
return new Promise((resolve, reject) => {
|
|
364
456
|
const results = []
|
|
365
|
-
//this.drive.createReadStream(fpath)
|
|
457
|
+
// this.drive.createReadStream(fpath)
|
|
366
458
|
rs.pipe(csv({ headers: headerSet.headerset, separator: headerSet.delimiter, skipLines: headerSet.dataline }))
|
|
367
459
|
.on('data', (data) => results.push(data))
|
|
368
460
|
.on('end', () => {
|
|
@@ -371,6 +463,22 @@ class HypDrive extends EventEmitter {
|
|
|
371
463
|
})
|
|
372
464
|
}
|
|
373
465
|
|
|
466
|
+
/**
|
|
467
|
+
* stream out line by line
|
|
468
|
+
* @method readCSVfileStream
|
|
469
|
+
*
|
|
470
|
+
*/
|
|
471
|
+
readCSVfileStream = async function (fpath) {
|
|
472
|
+
const rs = this.drive.createReadStream(fpath, { start: 0, end: 120 })
|
|
473
|
+
return new Promise((resolve, reject) => {
|
|
474
|
+
let results = []
|
|
475
|
+
rs.on('data', (data) => results.push(data.toString()))
|
|
476
|
+
rs.on('end', () => {
|
|
477
|
+
resolve(results)
|
|
478
|
+
})
|
|
479
|
+
})
|
|
480
|
+
}
|
|
481
|
+
|
|
374
482
|
/**
|
|
375
483
|
* replicate a hyperdrive
|
|
376
484
|
* @method hyperdriveReplicate
|
package/src/fileParser.js
CHANGED
|
@@ -143,7 +143,6 @@ FileParser.prototype.webCSVparse = function (fData) {
|
|
|
143
143
|
*
|
|
144
144
|
*/
|
|
145
145
|
FileParser.prototype.TEMPwebJSONparse = function (fjData) {
|
|
146
|
-
console.log(fjData)
|
|
147
146
|
let extractLabel = []
|
|
148
147
|
let extractCol = []
|
|
149
148
|
for (let df of fjData.content) {
|
|
@@ -315,20 +314,16 @@ FileParser.prototype.convertJSON = function (o, headerSet, results, source, newF
|
|
|
315
314
|
const datacolumn = o.data[0].info.datename
|
|
316
315
|
const flowList = []
|
|
317
316
|
for (const rs of results) {
|
|
318
|
-
// console.log(rs)
|
|
319
317
|
let timeLength = 0
|
|
320
318
|
// what length is date number? Need to make ms time standard to convert
|
|
321
319
|
if (rs[datacolumn].length === 10) {
|
|
322
|
-
// console.log('not ms time add 000')
|
|
323
320
|
timeLength = rs[datacolumn] * 1000
|
|
324
321
|
} else {
|
|
325
322
|
// console.log('assume ms time ')
|
|
326
323
|
timeLength = rs[datacolumn]
|
|
327
324
|
}
|
|
328
325
|
const dateFormat = new Date(timeLength)
|
|
329
|
-
// console.log(dateFormat)
|
|
330
326
|
const msDate = dateFormat.getTime()
|
|
331
|
-
// console.log(msDate)
|
|
332
327
|
rs[datacolumn] = msDate / 1000
|
|
333
328
|
flowList.push(rs)
|
|
334
329
|
}
|
package/src/index.js
CHANGED
|
@@ -76,9 +76,17 @@ class HolepunchWorker extends EventEmitter {
|
|
|
76
76
|
this.wsocket = ws
|
|
77
77
|
this.BeeData.setWebsocket(ws)
|
|
78
78
|
this.DriveFiles.setWebsocket(ws)
|
|
79
|
-
this.activateHypercores()
|
|
79
|
+
// this.activateHypercores()
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
+
/**
|
|
83
|
+
* active hypercores auth verified
|
|
84
|
+
* @method startStores
|
|
85
|
+
*
|
|
86
|
+
*/
|
|
87
|
+
startStores = function (ws) {
|
|
88
|
+
this.activateHypercores()
|
|
89
|
+
}
|
|
82
90
|
|
|
83
91
|
/**
|
|
84
92
|
* listen for outputs from workers
|
|
@@ -89,7 +97,7 @@ class HolepunchWorker extends EventEmitter {
|
|
|
89
97
|
this.Peers.on('peer-network', (data) => {
|
|
90
98
|
this.wsocket.send(JSON.stringify(data))
|
|
91
99
|
})
|
|
92
|
-
// peer connection active
|
|
100
|
+
// peer connection active for first time
|
|
93
101
|
this.Peers.on('peer-connect', (data) => {
|
|
94
102
|
let taskCheck = 0
|
|
95
103
|
let firstCheck = this.Peers.peerHolder[data]
|
|
@@ -103,10 +111,28 @@ class HolepunchWorker extends EventEmitter {
|
|
|
103
111
|
// if (this.Peers.peerHolder[data].data.boardID !== undefined) {
|
|
104
112
|
// any existing peers
|
|
105
113
|
let holderCheck = Object.keys(this.Peers.peerHolder)
|
|
114
|
+
// now first time, check if any message can be send?
|
|
106
115
|
if (holderCheck !== 0 && taskCheck === 0) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
116
|
+
console.log('HOP--write publis lib???')
|
|
117
|
+
// switch between
|
|
118
|
+
console.log(holderCheck)
|
|
119
|
+
console.log(taskCheck)
|
|
120
|
+
let peerFirstID = holderCheck[0]
|
|
121
|
+
console.log(peerFirstID)
|
|
122
|
+
console.log(this.Peers.peerHolder)
|
|
123
|
+
// which direction connection, client or from a peer on the network?
|
|
124
|
+
if (this.Peers.peerHolder[peerFirstID]?.data !== undefined) {
|
|
125
|
+
let dataLive = this.Peers.peerHolder[peerFirstID].data.name
|
|
126
|
+
if (dataLive === 'cue-space') {
|
|
127
|
+
this.Peers.writeToCueSpace(this.Peers.peerHolder[peerFirstID].publickey)
|
|
128
|
+
} else {
|
|
129
|
+
this.Peers.writeToPublicLibrary(data)
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
} else {
|
|
133
|
+
console.log('write to entwoir on start')
|
|
134
|
+
this.Peers.writeTonetwork(data)
|
|
135
|
+
}
|
|
110
136
|
}
|
|
111
137
|
|
|
112
138
|
})
|
|
@@ -114,6 +140,10 @@ class HolepunchWorker extends EventEmitter {
|
|
|
114
140
|
this.Peers.on('beebee-data', (data) => {
|
|
115
141
|
this.emit('peer-topeer', data)
|
|
116
142
|
})
|
|
143
|
+
this.Peers.on('cuespace-notification', (data) => {
|
|
144
|
+
console.log('cue space ntototoo')
|
|
145
|
+
this.emit('peer-cuespace', data)
|
|
146
|
+
})
|
|
117
147
|
// public library notification
|
|
118
148
|
this.Peers.on('publiclibrarynotification', (data) => {
|
|
119
149
|
this.BeeData.replicatePubliclibrary(data)
|
|
@@ -130,6 +160,9 @@ class HolepunchWorker extends EventEmitter {
|
|
|
130
160
|
this.warmPeers.push(peerId)
|
|
131
161
|
this.emit('peer-incoming', peerId)
|
|
132
162
|
})
|
|
163
|
+
this.DriveFiles.on('largefile-save', (data) => {
|
|
164
|
+
this.emit('drive-save-large', data)
|
|
165
|
+
})
|
|
133
166
|
}
|
|
134
167
|
|
|
135
168
|
/**
|
|
@@ -155,6 +188,21 @@ class HolepunchWorker extends EventEmitter {
|
|
|
155
188
|
this.warmPeers.push(message.data)
|
|
156
189
|
this.Peers.peerJoin(message.data)
|
|
157
190
|
}
|
|
191
|
+
} else if (message.task = 'cue-space') {
|
|
192
|
+
console.log('cus space send path')
|
|
193
|
+
console.log(peerMatch)
|
|
194
|
+
if (peerMatch === true) {
|
|
195
|
+
console.log('already connect space')
|
|
196
|
+
this.Peers.peerAlreadyJoinSetData(message.data)
|
|
197
|
+
this.Peers.writeToCueSpace(message.data.publickey)
|
|
198
|
+
} else {
|
|
199
|
+
console.log('first space send')
|
|
200
|
+
this.warmPeers.push(message.data)
|
|
201
|
+
this.Peers.peerJoin(message.data)
|
|
202
|
+
// this.Peers.peerAlreadyJoinSetData(message.data)
|
|
203
|
+
// this.Peers.writeToCueSpace(message.data.publickey)
|
|
204
|
+
console.log('space suce peepr ot sent')
|
|
205
|
+
}
|
|
158
206
|
} else if (message.task === 'peer-board') {
|
|
159
207
|
if (peerMatch === true) {
|
|
160
208
|
this.Peers.peerAlreadyJoinSetData(message.data)
|
package/src/peers.js
CHANGED
|
@@ -75,12 +75,16 @@ class NetworkPeers extends EventEmitter {
|
|
|
75
75
|
if (Buffer.isBuffer(data)) {
|
|
76
76
|
try {
|
|
77
77
|
let dataShareIn = JSON.parse(data.toString())
|
|
78
|
+
console.log('data in listen bubfferr')
|
|
79
|
+
console.log(dataShareIn.type)
|
|
78
80
|
if (dataShareIn.type === 'chart') {
|
|
79
81
|
this.emit('beebee-data', dataShareIn)
|
|
80
82
|
// need to look at NXP, modules and within for reference contracts.
|
|
81
83
|
// Need to replicate public library for contracts (repliate hyberbee)
|
|
82
84
|
// Need to ask for data source e.g. file (replicate hyberdrive)
|
|
83
85
|
// Lastly put together SafeFlowECS query to produce chart
|
|
86
|
+
} else if (dataShareIn.type === 'cue-space') {
|
|
87
|
+
this.emit('cuespace-notification', dataShareIn)
|
|
84
88
|
} else if (dataShareIn.type === 'public-library') {
|
|
85
89
|
this.emit('publiclibrarynotification', dataShareIn)
|
|
86
90
|
} else if (dataShareIn.type === 'peer') {
|
|
@@ -133,6 +137,25 @@ class NetworkPeers extends EventEmitter {
|
|
|
133
137
|
}
|
|
134
138
|
}
|
|
135
139
|
|
|
140
|
+
/**
|
|
141
|
+
* write message connect peers space
|
|
142
|
+
* @method writeToCueSpace
|
|
143
|
+
*
|
|
144
|
+
*/
|
|
145
|
+
writeToCueSpace = function (publickey) {
|
|
146
|
+
// check this peer has asked for space data
|
|
147
|
+
let connectTrue = publickey in this.peerConnect
|
|
148
|
+
let spaceTrue = publickey in this.peerHolder
|
|
149
|
+
if (connectTrue === true && spaceTrue === true) {
|
|
150
|
+
let libraryData = this.peerHolder[publickey]
|
|
151
|
+
let dataShare = {}
|
|
152
|
+
dataShare.data = libraryData.data
|
|
153
|
+
dataShare.type = 'cue-space'
|
|
154
|
+
this.peerConnect[publickey].write(JSON.stringify(dataShare))
|
|
155
|
+
} else {
|
|
156
|
+
console.log('no cuespace to write ie share with a peer')
|
|
157
|
+
}
|
|
158
|
+
}
|
|
136
159
|
|
|
137
160
|
|
|
138
161
|
/**
|
|
@@ -141,6 +164,8 @@ class NetworkPeers extends EventEmitter {
|
|
|
141
164
|
*
|
|
142
165
|
*/
|
|
143
166
|
peerJoin = function (peerContext) {
|
|
167
|
+
console.log('peerJOIn')
|
|
168
|
+
console.log(peerContext)
|
|
144
169
|
this.peerHolder[peerContext.publickey] = peerContext
|
|
145
170
|
const noisePublicKey = Buffer.from(peerContext.publickey, 'hex') // must be 32 bytes
|
|
146
171
|
if (noisePublicKey.length === 32) {
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import assert from 'assert'
|
|
2
|
+
import HolepunchData from '../src/index.js'
|
|
3
|
+
// need helpers prepare input to HP via (mock) HOP
|
|
4
|
+
|
|
5
|
+
describe('upload a large csv in chunks', function () {
|
|
6
|
+
it('read in file', async function () {
|
|
7
|
+
let dataAPI = new HolepunchData()
|
|
8
|
+
})
|
|
9
|
+
})
|