@fireproof/core 0.7.3-dev.2 → 0.8.0-dev.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blockstore.js +6 -0
- package/dist/crypto.js +5 -1
- package/dist/database.js +14 -7
- package/dist/fireproof.js +1 -0
- package/dist/loader.js +0 -4
- package/dist/prolly.js +1 -1
- package/dist/remote.js +102 -0
- package/dist/src/fireproof.d.ts +6 -5
- package/dist/src/fireproof.js +65878 -29050
- package/dist/src/fireproof.js.map +1 -1
- package/dist/src/fireproof.mjs +65878 -29050
- package/dist/src/fireproof.mjs.map +1 -1
- package/dist/storage/base.js +177 -262
- package/dist/storage/blocksToEncryptedCarBlock.js +144 -0
- package/dist/storage/browser.js +1 -1
- package/dist/storage/filesystem.js +2 -3
- package/dist/storage/rest.js +1 -2
- package/dist/storage/ucan.js +0 -40
- package/dist/storage/utils.js +144 -0
- package/dist/sync.js +1 -1
- package/dist/valet.js +9 -3
- package/package.json +3 -5
- package/src/blockstore.js +4 -0
- package/src/crypto.js +5 -1
- package/src/database.js +14 -9
- package/src/fireproof.js +1 -0
- package/src/loader.js +0 -5
- package/src/prolly.js +1 -1
- package/src/remote.js +113 -0
- package/src/storage/base.js +194 -275
- package/src/storage/browser.js +1 -1
- package/src/storage/filesystem.js +2 -3
- package/src/storage/rest.js +1 -2
- package/src/storage/ucan.js +0 -45
- package/src/storage/utils.js +152 -0
- package/src/sync.js +1 -1
- package/src/valet.js +9 -3
package/src/storage/ucan.js
CHANGED
@@ -1,45 +0,0 @@
|
|
1
|
-
import fetch from 'cross-fetch'
|
2
|
-
import { Base } from './base.js'
|
3
|
-
|
4
|
-
const defaultConfig = {
|
5
|
-
upload: () => {},
|
6
|
-
url: (cid) => `https://${cid}.ipfs.w3s.link/`
|
7
|
-
}
|
8
|
-
|
9
|
-
export class UCAN extends Base {
|
10
|
-
constructor (name, config = {}) {
|
11
|
-
super(name, Object.assign({}, defaultConfig, config))
|
12
|
-
}
|
13
|
-
|
14
|
-
async writeCars (cars) {
|
15
|
-
if (this.config.readonly) return
|
16
|
-
for (const { cid, bytes } of cars) {
|
17
|
-
console.log(`write UCAN ${cid}, ${bytes.length} bytes`)
|
18
|
-
const upCid = await this.config.upload(bytes)
|
19
|
-
console.log(`wrote UCAN ${cid}, ${upCid}`)
|
20
|
-
// if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
|
21
|
-
}
|
22
|
-
}
|
23
|
-
|
24
|
-
async readCar (carCid) {
|
25
|
-
const carURL = this.config.url(carCid)
|
26
|
-
const response = await fetch(carURL)
|
27
|
-
if (!response.ok) throw new Error(`An error occurred: ${response.statusText}`)
|
28
|
-
const got = await response.arrayBuffer()
|
29
|
-
return new Uint8Array(got)
|
30
|
-
}
|
31
|
-
|
32
|
-
async loadHeader (branch = 'main') {
|
33
|
-
return headerMock.get(branch)
|
34
|
-
}
|
35
|
-
|
36
|
-
async writeHeader (branch, header) {
|
37
|
-
if (this.config.readonly) return
|
38
|
-
const pHeader = this.prepareHeader(header)
|
39
|
-
// console.log('writeHeader rt', branch, pHeader)
|
40
|
-
|
41
|
-
headerMock.set(branch, pHeader)
|
42
|
-
}
|
43
|
-
}
|
44
|
-
|
45
|
-
const headerMock = new Map()
|
@@ -0,0 +1,152 @@
|
|
1
|
+
import * as CBW from '@ipld/car/buffer-writer'
|
2
|
+
import * as raw from 'multiformats/codecs/raw'
|
3
|
+
import { encrypt, decrypt } from '../crypto.js'
|
4
|
+
import { parse } from 'multiformats/link'
|
5
|
+
import { sha256 } from 'multiformats/hashes/sha2'
|
6
|
+
import * as Block from 'multiformats/block'
|
7
|
+
import { Buffer } from 'buffer'
|
8
|
+
// @ts-ignore
|
9
|
+
import { bf } from 'prolly-trees/utils'
|
10
|
+
// @ts-ignore
|
11
|
+
import { nocache as cache } from 'prolly-trees/cache'
|
12
|
+
const chunker = bf(30)
|
13
|
+
|
14
|
+
export async function getEmptyLoader () { // unused?
|
15
|
+
const theseWriteableBlocks = new VMemoryBlockstore()
|
16
|
+
return {
|
17
|
+
blocks: theseWriteableBlocks,
|
18
|
+
put: async (cid, bytes) => {
|
19
|
+
return await theseWriteableBlocks.put(cid, bytes)
|
20
|
+
},
|
21
|
+
get: async (cid) => {
|
22
|
+
const got = await theseWriteableBlocks.get(cid)
|
23
|
+
return got.bytes
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}
|
27
|
+
|
28
|
+
export class VMemoryBlockstore {
|
29
|
+
/** @type {Map<string, Uint8Array>} */
|
30
|
+
blocks = new Map()
|
31
|
+
instanceId = Math.random().toString(36).slice(2)
|
32
|
+
|
33
|
+
async get (cid) {
|
34
|
+
const bytes = this.blocks.get(cid.toString())
|
35
|
+
if (!bytes) throw new Error('block not found ' + cid.toString())
|
36
|
+
return { cid, bytes }
|
37
|
+
}
|
38
|
+
|
39
|
+
/**
|
40
|
+
* @param {any} cid
|
41
|
+
* @param {Uint8Array} bytes
|
42
|
+
*/
|
43
|
+
async put (cid, bytes) {
|
44
|
+
this.blocks.set(cid.toString(), bytes)
|
45
|
+
}
|
46
|
+
|
47
|
+
* entries () {
|
48
|
+
for (const [str, bytes] of this.blocks) {
|
49
|
+
yield { cid: parse(str), bytes }
|
50
|
+
}
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
export const blocksToCarBlock = async (rootCids, blocks) => {
|
55
|
+
// console.log('blocksToCarBlock', rootCids, blocks.constructor.name)
|
56
|
+
let size = 0
|
57
|
+
if (!Array.isArray(rootCids)) {
|
58
|
+
rootCids = [rootCids]
|
59
|
+
}
|
60
|
+
const headerSize = CBW.headerLength({ roots: rootCids })
|
61
|
+
size += headerSize
|
62
|
+
if (!Array.isArray(blocks)) {
|
63
|
+
blocks = Array.from(blocks.entries())
|
64
|
+
}
|
65
|
+
for (const { cid, bytes } of blocks) {
|
66
|
+
// console.log(cid, bytes)
|
67
|
+
size += CBW.blockLength({ cid, bytes })
|
68
|
+
}
|
69
|
+
const buffer = new Uint8Array(size)
|
70
|
+
const writer = await CBW.createWriter(buffer, { headerSize })
|
71
|
+
|
72
|
+
for (const cid of rootCids) {
|
73
|
+
writer.addRoot(cid)
|
74
|
+
}
|
75
|
+
|
76
|
+
for (const { cid, bytes } of blocks) {
|
77
|
+
writer.write({ cid, bytes })
|
78
|
+
}
|
79
|
+
await writer.close()
|
80
|
+
return await Block.encode({ value: writer.bytes, hasher: sha256, codec: raw })
|
81
|
+
}
|
82
|
+
|
83
|
+
export const blocksToEncryptedCarBlock = async (innerBlockStoreClockRootCid, blocks, keyMaterial, cids) => {
|
84
|
+
const encryptionKey = Buffer.from(keyMaterial, 'hex')
|
85
|
+
const encryptedBlocks = []
|
86
|
+
const theCids = cids
|
87
|
+
// console.trace('blocksToEncryptedCarBlock', blocks)
|
88
|
+
// for (const { cid } of blocks.entries()) {
|
89
|
+
// theCids.push(cid.toString())
|
90
|
+
// }
|
91
|
+
// console.log(
|
92
|
+
// 'encrypting',
|
93
|
+
// theCids.length,
|
94
|
+
// 'blocks',
|
95
|
+
// theCids.includes(innerBlockStoreClockRootCid.toString()),
|
96
|
+
// keyMaterial
|
97
|
+
// )
|
98
|
+
// console.log('cids', theCids, innerBlockStoreClockRootCid.toString())
|
99
|
+
let last
|
100
|
+
for await (const block of encrypt({
|
101
|
+
cids: theCids,
|
102
|
+
get: async (cid) => {
|
103
|
+
// console.log('getencrypt', cid)
|
104
|
+
const got = blocks.get(cid)
|
105
|
+
// console.log('got', got)
|
106
|
+
return got.block ? ({ cid, bytes: got.block }) : got
|
107
|
+
},
|
108
|
+
key: encryptionKey,
|
109
|
+
hasher: sha256,
|
110
|
+
chunker,
|
111
|
+
cache,
|
112
|
+
// codec: dagcbor, // should be crypto?
|
113
|
+
root: innerBlockStoreClockRootCid
|
114
|
+
})) {
|
115
|
+
encryptedBlocks.push(block)
|
116
|
+
last = block
|
117
|
+
}
|
118
|
+
// console.log('last', last.cid.toString(), 'for clock', innerBlockStoreClockRootCid.toString())
|
119
|
+
const encryptedCar = await blocksToCarBlock(last.cid, encryptedBlocks)
|
120
|
+
return encryptedCar
|
121
|
+
}
|
122
|
+
|
123
|
+
// { root, get, key, cache, chunker, hasher }
|
124
|
+
const memoizeDecryptedCarBlocks = new Map()
|
125
|
+
export const blocksFromEncryptedCarBlock = async (cid, get, keyMaterial) => {
|
126
|
+
if (memoizeDecryptedCarBlocks.has(cid.toString())) {
|
127
|
+
return memoizeDecryptedCarBlocks.get(cid.toString())
|
128
|
+
} else {
|
129
|
+
const blocksPromise = (async () => {
|
130
|
+
const decryptionKey = Buffer.from(keyMaterial, 'hex')
|
131
|
+
// console.log('decrypting', keyMaterial, cid.toString())
|
132
|
+
const cids = new Set()
|
133
|
+
const decryptedBlocks = []
|
134
|
+
for await (const block of decrypt({
|
135
|
+
root: cid,
|
136
|
+
get,
|
137
|
+
key: decryptionKey,
|
138
|
+
chunker,
|
139
|
+
hasher: sha256,
|
140
|
+
cache
|
141
|
+
// codec: dagcbor
|
142
|
+
})) {
|
143
|
+
// console.log('decrypted', block.cid.toString())
|
144
|
+
decryptedBlocks.push(block)
|
145
|
+
cids.add(block.cid.toString())
|
146
|
+
}
|
147
|
+
return { blocks: decryptedBlocks, cids }
|
148
|
+
})()
|
149
|
+
memoizeDecryptedCarBlocks.set(cid.toString(), blocksPromise)
|
150
|
+
return blocksPromise
|
151
|
+
}
|
152
|
+
}
|
package/src/sync.js
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
import SimplePeer from 'simple-peer'
|
2
2
|
import { parseCID } from './database.js'
|
3
3
|
import { decodeEventBlock } from './clock.js'
|
4
|
-
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/
|
4
|
+
import { blocksToCarBlock, blocksToEncryptedCarBlock } from './storage/utils.js'
|
5
5
|
import { CarReader } from '@ipld/car'
|
6
6
|
|
7
7
|
/**
|
package/src/valet.js
CHANGED
@@ -79,16 +79,22 @@ export class Valet {
|
|
79
79
|
if (this.secondary) {
|
80
80
|
// console.log('getValetBlock secondary', dataCID)
|
81
81
|
try {
|
82
|
+
// eslint-disable-next-line
|
82
83
|
const { block, reader } = await this.secondary.getLoaderBlock(dataCID)
|
84
|
+
const writeableCarReader = await this.primary.getWriteableCarReader(reader)
|
85
|
+
// console.log('getValetBlock secondary', dataCID, block.length)
|
86
|
+
// eslint-disable-next-line
|
83
87
|
const cids = new Set()
|
84
88
|
for await (const { cid } of reader.entries()) {
|
85
89
|
// console.log(cid, bytes)
|
86
90
|
cids.add(cid.toString())
|
87
91
|
}
|
88
|
-
reader.get = reader.gat // some consumers prefer get
|
92
|
+
// reader.get = reader.gat // some consumers prefer get
|
89
93
|
// console.log('replicating', reader.root)
|
90
|
-
|
91
|
-
|
94
|
+
writeableCarReader.lastCid = reader.root.cid
|
95
|
+
writeableCarReader.head = []
|
96
|
+
await this.primary.parkCar(writeableCarReader, cids).catch(e => console.error('parkCar error', e))
|
97
|
+
// console.log('FIX THIS', did)
|
92
98
|
return block
|
93
99
|
} catch (e) {
|
94
100
|
// console.log('getValetBlock secondary error', e)
|