@web3-storage/pail 0.6.0-alpha.4 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +208 -0
- package/dist/scripts/randomcid.d.ts +2 -0
- package/dist/scripts/randomcid.d.ts.map +1 -0
- package/dist/scripts/randomcid.js +10 -0
- package/dist/scripts/words/gen.d.ts +2 -0
- package/dist/scripts/words/gen.d.ts.map +1 -0
- package/dist/scripts/words/gen.js +51 -0
- package/dist/src/api.d.ts +4 -2
- package/dist/src/api.d.ts.map +1 -1
- package/dist/src/api.js +1 -0
- package/dist/src/batch/api.d.ts +2 -4
- package/dist/src/batch/api.d.ts.map +1 -1
- package/dist/src/batch/api.js +1 -0
- package/dist/src/batch/index.d.ts +2 -53
- package/dist/src/batch/index.d.ts.map +1 -1
- package/dist/src/batch/index.js +241 -0
- package/dist/src/batch/shard.d.ts +1 -1
- package/dist/src/batch/shard.d.ts.map +1 -1
- package/dist/src/batch/shard.js +12 -0
- package/dist/src/block.d.ts +3 -3
- package/dist/src/block.d.ts.map +1 -1
- package/dist/src/block.js +66 -0
- package/dist/src/clock/api.d.ts +1 -1
- package/dist/src/clock/api.d.ts.map +1 -1
- package/dist/src/clock/api.js +1 -0
- package/dist/src/clock/index.d.ts +2 -2
- package/dist/src/clock/index.d.ts.map +1 -1
- package/dist/src/clock/index.js +199 -0
- package/dist/src/crdt/api.d.ts +2 -2
- package/dist/src/crdt/api.d.ts.map +1 -1
- package/dist/src/crdt/api.js +1 -0
- package/dist/src/crdt/batch/api.d.ts +1 -1
- package/dist/src/crdt/batch/api.d.ts.map +1 -1
- package/dist/src/crdt/batch/api.js +1 -0
- package/dist/src/crdt/batch/index.d.ts.map +1 -1
- package/dist/src/crdt/batch/index.js +140 -0
- package/dist/src/crdt/index.d.ts +2 -4
- package/dist/src/crdt/index.d.ts.map +1 -1
- package/dist/src/crdt/index.js +344 -0
- package/dist/src/diff.d.ts +3 -3
- package/dist/src/diff.d.ts.map +1 -1
- package/dist/src/diff.js +151 -0
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +356 -0
- package/dist/src/merge.d.ts.map +1 -1
- package/dist/src/merge.js +42 -0
- package/dist/src/shard.d.ts +4 -4
- package/dist/src/shard.d.ts.map +1 -1
- package/dist/src/shard.js +166 -0
- package/dist/test/batch.test.d.ts +2 -0
- package/dist/test/batch.test.d.ts.map +1 -0
- package/dist/test/batch.test.js +126 -0
- package/dist/test/clock.test.d.ts +2 -0
- package/dist/test/clock.test.d.ts.map +1 -0
- package/dist/test/clock.test.js +244 -0
- package/dist/test/crdt.test.d.ts +2 -0
- package/dist/test/crdt.test.d.ts.map +1 -0
- package/dist/test/crdt.test.js +271 -0
- package/dist/test/del.test.d.ts +2 -0
- package/dist/test/del.test.d.ts.map +1 -0
- package/dist/test/del.test.js +21 -0
- package/dist/test/diff.test.d.ts +2 -0
- package/dist/test/diff.test.d.ts.map +1 -0
- package/dist/test/diff.test.js +38 -0
- package/dist/test/entries.test.d.ts +2 -0
- package/dist/test/entries.test.d.ts.map +1 -0
- package/dist/test/entries.test.js +210 -0
- package/dist/test/get.test.d.ts +2 -0
- package/dist/test/get.test.d.ts.map +1 -0
- package/dist/test/get.test.js +26 -0
- package/dist/test/helpers.d.ts +35 -0
- package/dist/test/helpers.d.ts.map +1 -0
- package/dist/test/helpers.js +168 -0
- package/dist/test/put.test.d.ts +2 -0
- package/dist/test/put.test.d.ts.map +1 -0
- package/dist/test/put.test.js +165 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/vitest.config.d.ts +3 -0
- package/dist/vitest.config.d.ts.map +1 -0
- package/dist/vitest.config.js +13 -0
- package/package.json +47 -92
- package/scripts/propernames/gen.sh +3 -0
- package/cli.js +0 -260
- package/dist/src/batch.d.ts +0 -62
- package/dist/src/batch.d.ts.map +0 -1
- package/dist/src/crdt/batch.d.ts +0 -3
- package/dist/src/crdt/batch.d.ts.map +0 -1
- package/src/api.js +0 -1
- package/src/api.ts +0 -90
- package/src/batch/api.js +0 -1
- package/src/batch/api.ts +0 -59
- package/src/batch/index.js +0 -248
- package/src/batch/index2.js +0 -0
- package/src/batch/shard.js +0 -13
- package/src/block.js +0 -75
- package/src/clock/api.js +0 -1
- package/src/clock/api.ts +0 -12
- package/src/clock/index.js +0 -182
- package/src/crdt/api.js +0 -1
- package/src/crdt/api.ts +0 -33
- package/src/crdt/batch/api.js +0 -1
- package/src/crdt/batch/api.ts +0 -30
- package/src/crdt/batch/index.js +0 -156
- package/src/crdt/index.js +0 -355
- package/src/diff.js +0 -151
- package/src/index.js +0 -406
- package/src/merge.js +0 -43
- package/src/shard.js +0 -180
package/package.json
CHANGED
|
@@ -1,132 +1,73 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@web3-storage/pail",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.2",
|
|
4
4
|
"description": "DAG based key value store.",
|
|
5
|
-
"main": "src/index.js",
|
|
5
|
+
"main": "./dist/src/index.js",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"types": "./dist/src/index.d.ts",
|
|
8
|
-
"typesVersions": {
|
|
9
|
-
"*": {
|
|
10
|
-
"*": [
|
|
11
|
-
"dist/*"
|
|
12
|
-
],
|
|
13
|
-
"dist/src/index.d.ts": [
|
|
14
|
-
"dist/src/index.d.ts"
|
|
15
|
-
],
|
|
16
|
-
"api": [
|
|
17
|
-
"dist/src/api.d.ts"
|
|
18
|
-
],
|
|
19
|
-
"batch": [
|
|
20
|
-
"dist/src/batch/index.d.ts"
|
|
21
|
-
],
|
|
22
|
-
"batch/api": [
|
|
23
|
-
"dist/src/batch/api.d.ts"
|
|
24
|
-
],
|
|
25
|
-
"block": [
|
|
26
|
-
"dist/src/block.d.ts"
|
|
27
|
-
],
|
|
28
|
-
"clock": [
|
|
29
|
-
"dist/src/clock/index.d.ts"
|
|
30
|
-
],
|
|
31
|
-
"clock/api": [
|
|
32
|
-
"dist/src/clock/api.d.ts"
|
|
33
|
-
],
|
|
34
|
-
"crdt": [
|
|
35
|
-
"dist/src/crdt/index.d.ts"
|
|
36
|
-
],
|
|
37
|
-
"crdt/api": [
|
|
38
|
-
"dist/src/crdt/api.d.ts"
|
|
39
|
-
],
|
|
40
|
-
"crdt/batch": [
|
|
41
|
-
"dist/src/crdt/batch/index.d.ts"
|
|
42
|
-
],
|
|
43
|
-
"crdt/batch/api": [
|
|
44
|
-
"dist/src/crdt/batch/api.d.ts"
|
|
45
|
-
],
|
|
46
|
-
"diff": [
|
|
47
|
-
"dist/src/diff.d.ts"
|
|
48
|
-
],
|
|
49
|
-
"link": [
|
|
50
|
-
"dist/src/link.d.ts"
|
|
51
|
-
],
|
|
52
|
-
"merge": [
|
|
53
|
-
"dist/src/merge.d.ts"
|
|
54
|
-
],
|
|
55
|
-
"shard": [
|
|
56
|
-
"dist/src/shard.d.ts"
|
|
57
|
-
]
|
|
58
|
-
}
|
|
59
|
-
},
|
|
60
8
|
"exports": {
|
|
61
9
|
".": {
|
|
62
10
|
"types": "./dist/src/index.d.ts",
|
|
63
|
-
"import": "./src/index.js"
|
|
11
|
+
"import": "./dist/src/index.js"
|
|
64
12
|
},
|
|
65
13
|
"./api": {
|
|
66
14
|
"types": "./dist/src/api.d.ts",
|
|
67
|
-
"import": "./src/api.js"
|
|
15
|
+
"import": "./dist/src/api.js"
|
|
68
16
|
},
|
|
69
17
|
"./batch": {
|
|
70
18
|
"types": "./dist/src/batch/index.d.ts",
|
|
71
|
-
"import": "./src/batch/index.js"
|
|
19
|
+
"import": "./dist/src/batch/index.js"
|
|
72
20
|
},
|
|
73
21
|
"./batch/api": {
|
|
74
22
|
"types": "./dist/src/batch/api.d.ts",
|
|
75
|
-
"import": "./src/batch/api.js"
|
|
23
|
+
"import": "./dist/src/batch/api.js"
|
|
76
24
|
},
|
|
77
25
|
"./block": {
|
|
78
26
|
"types": "./dist/src/block.d.ts",
|
|
79
|
-
"import": "./src/block.js"
|
|
27
|
+
"import": "./dist/src/block.js"
|
|
80
28
|
},
|
|
81
29
|
"./clock": {
|
|
82
30
|
"types": "./dist/src/clock/index.d.ts",
|
|
83
|
-
"import": "./src/clock/index.js"
|
|
31
|
+
"import": "./dist/src/clock/index.js"
|
|
84
32
|
},
|
|
85
33
|
"./clock/api": {
|
|
86
34
|
"types": "./dist/src/clock/api.d.ts",
|
|
87
|
-
"import": "./src/clock/api.js"
|
|
35
|
+
"import": "./dist/src/clock/api.js"
|
|
88
36
|
},
|
|
89
37
|
"./crdt": {
|
|
90
38
|
"types": "./dist/src/crdt/index.d.ts",
|
|
91
|
-
"import": "./src/crdt/index.js"
|
|
39
|
+
"import": "./dist/src/crdt/index.js"
|
|
92
40
|
},
|
|
93
41
|
"./crdt/api": {
|
|
94
42
|
"types": "./dist/src/crdt/api.d.ts",
|
|
95
|
-
"import": "./src/crdt/api.js"
|
|
43
|
+
"import": "./dist/src/crdt/api.js"
|
|
96
44
|
},
|
|
97
45
|
"./crdt/batch": {
|
|
98
46
|
"types": "./dist/src/crdt/batch/index.d.ts",
|
|
99
|
-
"import": "./src/crdt/batch/index.js"
|
|
47
|
+
"import": "./dist/src/crdt/batch/index.js"
|
|
100
48
|
},
|
|
101
49
|
"./crdt/batch/api": {
|
|
102
50
|
"types": "./dist/src/crdt/batch/api.d.ts",
|
|
103
|
-
"import": "./src/crdt/batch/api.js"
|
|
51
|
+
"import": "./dist/src/crdt/batch/api.js"
|
|
104
52
|
},
|
|
105
53
|
"./diff": {
|
|
106
54
|
"types": "./dist/src/diff.d.ts",
|
|
107
|
-
"import": "./src/diff.js"
|
|
108
|
-
},
|
|
109
|
-
"./link": {
|
|
110
|
-
"types": "./dist/src/link.d.ts",
|
|
111
|
-
"import": "./src/link.js"
|
|
55
|
+
"import": "./dist/src/diff.js"
|
|
112
56
|
},
|
|
113
57
|
"./merge": {
|
|
114
58
|
"types": "./dist/src/merge.d.ts",
|
|
115
|
-
"import": "./src/merge.js"
|
|
59
|
+
"import": "./dist/src/merge.js"
|
|
116
60
|
},
|
|
117
61
|
"./shard": {
|
|
118
62
|
"types": "./dist/src/shard.d.ts",
|
|
119
|
-
"import": "./src/shard.js"
|
|
63
|
+
"import": "./dist/src/shard.js"
|
|
120
64
|
}
|
|
121
65
|
},
|
|
122
66
|
"bin": {
|
|
123
|
-
"pail": "./cli.js"
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
"
|
|
127
|
-
"test": "mocha test/*.test.js",
|
|
128
|
-
"coverage": "c8 -r html -r text npm test",
|
|
129
|
-
"lint": "standard"
|
|
67
|
+
"pail": "./dist/cli.js",
|
|
68
|
+
"pail-propernames": "./scripts/propernames/gen.sh",
|
|
69
|
+
"pail-gen": "./dist/scripts/words/gen.js",
|
|
70
|
+
"pail-randomcid": "./dist/scripts/randomcid.js"
|
|
130
71
|
},
|
|
131
72
|
"keywords": [
|
|
132
73
|
"bucket",
|
|
@@ -139,23 +80,25 @@
|
|
|
139
80
|
"author": "Alan Shaw",
|
|
140
81
|
"license": "Apache-2.0 OR MIT",
|
|
141
82
|
"files": [
|
|
142
|
-
"src",
|
|
143
83
|
"dist"
|
|
144
84
|
],
|
|
145
85
|
"dependencies": {
|
|
146
|
-
"@ipld/
|
|
147
|
-
"@ipld/dag-cbor": "^9.0.6",
|
|
148
|
-
"archy": "^1.0.0",
|
|
86
|
+
"@ipld/dag-cbor": "^9.2.2",
|
|
149
87
|
"cli-color": "^2.0.3",
|
|
150
|
-
"
|
|
88
|
+
"@ipld/car": "^5.4.0",
|
|
89
|
+
"archy": "^1.0.0",
|
|
90
|
+
"carstream": "^2.0.0",
|
|
91
|
+
"multiformats": "^13.3.2",
|
|
151
92
|
"sade": "^1.8.1"
|
|
152
93
|
},
|
|
153
94
|
"devDependencies": {
|
|
154
|
-
"
|
|
155
|
-
"
|
|
156
|
-
"
|
|
157
|
-
"
|
|
158
|
-
"
|
|
95
|
+
"@types/archy": "^0.0.36",
|
|
96
|
+
"@types/cli-color": "^2.0.6",
|
|
97
|
+
"@types/node": "^22.13.17",
|
|
98
|
+
"@vitest/coverage-v8": "^3.1.1",
|
|
99
|
+
"standard": "^17.1.2",
|
|
100
|
+
"typescript": "^5.8.2",
|
|
101
|
+
"vitest": "^3.1.1"
|
|
159
102
|
},
|
|
160
103
|
"repository": {
|
|
161
104
|
"type": "git",
|
|
@@ -166,8 +109,20 @@
|
|
|
166
109
|
},
|
|
167
110
|
"homepage": "https://github.com/web3-storage/pail#readme",
|
|
168
111
|
"standard": {
|
|
169
|
-
"
|
|
170
|
-
"
|
|
112
|
+
"globals": [
|
|
113
|
+
"describe",
|
|
114
|
+
"before",
|
|
115
|
+
"after",
|
|
116
|
+
"beforeEach",
|
|
117
|
+
"afterEach",
|
|
118
|
+
"it",
|
|
119
|
+
"assert"
|
|
171
120
|
]
|
|
121
|
+
},
|
|
122
|
+
"scripts": {
|
|
123
|
+
"build": "tsc --build",
|
|
124
|
+
"test": "vitest --run",
|
|
125
|
+
"coverage": "vitest --run --coverage",
|
|
126
|
+
"lint": "standard"
|
|
172
127
|
}
|
|
173
|
-
}
|
|
128
|
+
}
|
package/cli.js
DELETED
|
@@ -1,260 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import fs from 'fs'
|
|
3
|
-
import os from 'os'
|
|
4
|
-
import { join } from 'path'
|
|
5
|
-
import { Readable } from 'stream'
|
|
6
|
-
import sade from 'sade'
|
|
7
|
-
import { CID } from 'multiformats/cid'
|
|
8
|
-
import { CarIndexedReader, CarReader, CarWriter } from '@ipld/car'
|
|
9
|
-
import clc from 'cli-color'
|
|
10
|
-
import archy from 'archy'
|
|
11
|
-
// eslint-disable-next-line no-unused-vars
|
|
12
|
-
import * as API from './src/api.js'
|
|
13
|
-
import { put, get, del, entries } from './src/v1/index.js'
|
|
14
|
-
import { ShardFetcher, ShardBlock } from './src/v1/shard.js'
|
|
15
|
-
import { MaxShardSize } from './src/shard.js'
|
|
16
|
-
import { difference } from './src/diff.js'
|
|
17
|
-
import { merge } from './src/merge.js'
|
|
18
|
-
|
|
19
|
-
const cli = sade('pail')
|
|
20
|
-
.option('--path', 'Path to data store.', './pail.car')
|
|
21
|
-
|
|
22
|
-
cli.command('put <key> <value>')
|
|
23
|
-
.describe('Put a value (a CID) for the given key. If the key exists it\'s value is overwritten.')
|
|
24
|
-
.alias('set')
|
|
25
|
-
.option('--max-shard-size', 'Maximum shard size in bytes.', MaxShardSize)
|
|
26
|
-
.action(async (key, value, opts) => {
|
|
27
|
-
const maxShardSize = opts['max-shard-size'] ?? MaxShardSize
|
|
28
|
-
const blocks = await openPail(opts.path, { maxSize: maxShardSize })
|
|
29
|
-
const roots = await blocks.getRoots()
|
|
30
|
-
// @ts-expect-error
|
|
31
|
-
const { root, additions, removals } = await put(blocks, roots[0], key, CID.parse(value))
|
|
32
|
-
await updatePail(opts.path, blocks, root, { additions, removals })
|
|
33
|
-
|
|
34
|
-
console.log(clc.red(`--- ${roots[0]}`))
|
|
35
|
-
console.log(clc.green(`+++ ${root}`))
|
|
36
|
-
console.log(clc.magenta('@@ -1 +1 @@'))
|
|
37
|
-
additions.forEach(b => console.log(clc.green(`+${b.cid}`)))
|
|
38
|
-
removals.forEach(b => console.log(clc.red(`-${b.cid}`)))
|
|
39
|
-
await closePail(blocks)
|
|
40
|
-
})
|
|
41
|
-
|
|
42
|
-
cli.command('get <key>')
|
|
43
|
-
.describe('Get the stored value for the given key from the pail. If the key is not found, `undefined` is returned.')
|
|
44
|
-
.action(async (key, opts) => {
|
|
45
|
-
const blocks = await openPail(opts.path)
|
|
46
|
-
// @ts-expect-error
|
|
47
|
-
const value = await get(blocks, (await blocks.getRoots())[0], key)
|
|
48
|
-
if (value) console.log(value.toString())
|
|
49
|
-
await closePail(blocks)
|
|
50
|
-
})
|
|
51
|
-
|
|
52
|
-
cli.command('del <key>')
|
|
53
|
-
.describe('Delete the value for the given key from the pail. If the key is not found no operation occurs.')
|
|
54
|
-
.alias('delete', 'rm', 'remove')
|
|
55
|
-
.action(async (key, opts) => {
|
|
56
|
-
const blocks = await openPail(opts.path)
|
|
57
|
-
const roots = await blocks.getRoots()
|
|
58
|
-
// @ts-expect-error
|
|
59
|
-
const { root, additions, removals } = await del(blocks, roots[0], key)
|
|
60
|
-
await updatePail(opts.path, blocks, root, { additions, removals })
|
|
61
|
-
|
|
62
|
-
console.log(clc.red(`--- ${roots[0]}`))
|
|
63
|
-
console.log(clc.green(`+++ ${root}`))
|
|
64
|
-
console.log(clc.magenta('@@ -1 +1 @@'))
|
|
65
|
-
additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
|
|
66
|
-
removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
|
|
67
|
-
await closePail(blocks)
|
|
68
|
-
})
|
|
69
|
-
|
|
70
|
-
cli.command('ls')
|
|
71
|
-
.describe('List entries in the pail.')
|
|
72
|
-
.alias('list')
|
|
73
|
-
.option('-p, --prefix', 'Key prefix to filter by.')
|
|
74
|
-
.option('--json', 'Format output as newline delimted JSON.')
|
|
75
|
-
.action(async (opts) => {
|
|
76
|
-
const blocks = await openPail(opts.path)
|
|
77
|
-
const root = (await blocks.getRoots())[0]
|
|
78
|
-
let n = 0
|
|
79
|
-
// @ts-expect-error
|
|
80
|
-
for await (const [k, v] of entries(blocks, root, { prefix: opts.prefix })) {
|
|
81
|
-
console.log(opts.json ? JSON.stringify({ key: k, value: v.toString() }) : `${k}\t${v}`)
|
|
82
|
-
n++
|
|
83
|
-
}
|
|
84
|
-
if (!opts.json) console.log(`total ${n}`)
|
|
85
|
-
await closePail(blocks)
|
|
86
|
-
})
|
|
87
|
-
|
|
88
|
-
cli.command('tree')
|
|
89
|
-
.describe('Visualise the pail.')
|
|
90
|
-
.action(async (opts) => {
|
|
91
|
-
const blocks = await openPail(opts.path)
|
|
92
|
-
const root = (await blocks.getRoots())[0]
|
|
93
|
-
// @ts-expect-error
|
|
94
|
-
const shards = new ShardFetcher(blocks)
|
|
95
|
-
// @ts-expect-error
|
|
96
|
-
const rshard = await shards.get(root)
|
|
97
|
-
|
|
98
|
-
/** @type {archy.Data} */
|
|
99
|
-
const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] }
|
|
100
|
-
|
|
101
|
-
/** @param {API.ShardEntry} entry */
|
|
102
|
-
const getData = async ([k, v]) => {
|
|
103
|
-
if (!Array.isArray(v)) {
|
|
104
|
-
return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] }
|
|
105
|
-
}
|
|
106
|
-
/** @type {archy.Data} */
|
|
107
|
-
const data = { label: `Key(${clc.magenta(k)})`, nodes: [] }
|
|
108
|
-
if (v[1]) data.nodes?.push({ label: `Value(${clc.cyan(v[1])})` })
|
|
109
|
-
const blk = await shards.get(v[0])
|
|
110
|
-
data.nodes?.push({
|
|
111
|
-
label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`,
|
|
112
|
-
nodes: await Promise.all(blk.value.entries.map(e => getData(e)))
|
|
113
|
-
})
|
|
114
|
-
return data
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
for (const entry of rshard.value.entries) {
|
|
118
|
-
archyRoot.nodes?.push(await getData(entry))
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
console.log(archy(archyRoot))
|
|
122
|
-
await closePail(blocks)
|
|
123
|
-
})
|
|
124
|
-
|
|
125
|
-
cli.command('diff <path>')
|
|
126
|
-
.describe('Find the differences between this pail and the passed pail.')
|
|
127
|
-
.option('-k, --keys', 'Output key/value diff.')
|
|
128
|
-
.action(async (path, opts) => {
|
|
129
|
-
const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)])
|
|
130
|
-
const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => {
|
|
131
|
-
return /** @type {API.ShardLink} */((await blocks.getRoots())[0])
|
|
132
|
-
}))
|
|
133
|
-
if (aroot.toString() === broot.toString()) return
|
|
134
|
-
|
|
135
|
-
const fetcher = {
|
|
136
|
-
async get (cid) {
|
|
137
|
-
const blk = await ablocks.get(cid)
|
|
138
|
-
if (blk) return blk
|
|
139
|
-
return bblocks.get(cid)
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
// @ts-expect-error CarReader is not BlockFetcher
|
|
143
|
-
const { shards: { additions, removals }, keys } = await difference(fetcher, aroot, broot)
|
|
144
|
-
|
|
145
|
-
console.log(clc.red(`--- ${aroot}`))
|
|
146
|
-
console.log(clc.green(`+++ ${broot}`))
|
|
147
|
-
console.log(clc.magenta('@@ -1 +1 @@'))
|
|
148
|
-
|
|
149
|
-
if (opts.keys) {
|
|
150
|
-
keys.forEach(([k, v]) => {
|
|
151
|
-
if (v[0] != null) console.log(clc.red(`- ${k}\t${v[0]}`))
|
|
152
|
-
if (v[1] != null) console.log(clc.green(`+ ${k}\t${v[1]}`))
|
|
153
|
-
})
|
|
154
|
-
} else {
|
|
155
|
-
additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
|
|
156
|
-
removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
await Promise.all([closePail(ablocks), closePail(bblocks)])
|
|
160
|
-
})
|
|
161
|
-
|
|
162
|
-
cli.command('merge <path>')
|
|
163
|
-
.describe('Merge the passed pail into this pail.')
|
|
164
|
-
.action(async (path, opts) => {
|
|
165
|
-
const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)])
|
|
166
|
-
const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => {
|
|
167
|
-
return /** @type {API.ShardLink} */((await blocks.getRoots())[0])
|
|
168
|
-
}))
|
|
169
|
-
if (aroot.toString() === broot.toString()) return
|
|
170
|
-
|
|
171
|
-
const fetcher = {
|
|
172
|
-
async get (cid) {
|
|
173
|
-
const blk = await ablocks.get(cid)
|
|
174
|
-
if (blk) return blk
|
|
175
|
-
return bblocks.get(cid)
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
// @ts-expect-error CarReader is not BlockFetcher
|
|
179
|
-
const { root, additions, removals } = await merge(fetcher, aroot, [broot])
|
|
180
|
-
|
|
181
|
-
await updatePail(opts.path, ablocks, root, { additions, removals })
|
|
182
|
-
|
|
183
|
-
console.log(clc.red(`--- ${aroot}`))
|
|
184
|
-
console.log(clc.green(`+++ ${root}`))
|
|
185
|
-
console.log(clc.magenta('@@ -1 +1 @@'))
|
|
186
|
-
additions.forEach(b => console.log(clc.green(`+ ${b.cid}`)))
|
|
187
|
-
removals.forEach(b => console.log(clc.red(`- ${b.cid}`)))
|
|
188
|
-
|
|
189
|
-
await Promise.all([closePail(ablocks), closePail(bblocks)])
|
|
190
|
-
})
|
|
191
|
-
|
|
192
|
-
cli.parse(process.argv)
|
|
193
|
-
|
|
194
|
-
/**
|
|
195
|
-
* @param {string} path
|
|
196
|
-
* @param {{ maxSize?: number }} [config]
|
|
197
|
-
* @returns {Promise<import('@ipld/car/api').CarReader>}
|
|
198
|
-
*/
|
|
199
|
-
async function openPail (path, config) {
|
|
200
|
-
try {
|
|
201
|
-
return await CarIndexedReader.fromFile(path)
|
|
202
|
-
} catch (err) {
|
|
203
|
-
if (err.code !== 'ENOENT') throw new Error('failed to open bucket', { cause: err })
|
|
204
|
-
const rootblk = await ShardBlock.create(config)
|
|
205
|
-
const { writer, out } = CarWriter.create(rootblk.cid)
|
|
206
|
-
writer.put(rootblk)
|
|
207
|
-
writer.close()
|
|
208
|
-
return CarReader.fromIterable(out)
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
/** @param {import('@ipld/car/api').CarReader} reader */
|
|
213
|
-
async function closePail (reader) {
|
|
214
|
-
if (reader instanceof CarIndexedReader) {
|
|
215
|
-
await reader.close()
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
/**
|
|
220
|
-
* @param {string} path
|
|
221
|
-
* @param {import('@ipld/car/api').CarReader} reader
|
|
222
|
-
* @param {API.ShardLink} root
|
|
223
|
-
* @param {API.ShardDiff} diff
|
|
224
|
-
*/
|
|
225
|
-
async function updatePail (path, reader, root, { additions, removals }) {
|
|
226
|
-
// @ts-expect-error
|
|
227
|
-
const { writer, out } = CarWriter.create(root)
|
|
228
|
-
const tmp = join(os.tmpdir(), `pail${Date.now()}.car`)
|
|
229
|
-
|
|
230
|
-
const finishPromise = new Promise(resolve => {
|
|
231
|
-
Readable.from(out).pipe(fs.createWriteStream(tmp)).on('finish', resolve)
|
|
232
|
-
})
|
|
233
|
-
|
|
234
|
-
// put new blocks
|
|
235
|
-
for (const b of additions) {
|
|
236
|
-
await writer.put(b)
|
|
237
|
-
}
|
|
238
|
-
// put old blocks without removals
|
|
239
|
-
for await (const b of reader.blocks()) {
|
|
240
|
-
if (removals.some(r => b.cid.toString() === r.cid.toString())) {
|
|
241
|
-
continue
|
|
242
|
-
}
|
|
243
|
-
await writer.put(b)
|
|
244
|
-
}
|
|
245
|
-
await writer.close()
|
|
246
|
-
await finishPromise
|
|
247
|
-
|
|
248
|
-
const old = `${path}-${new Date().toISOString()}`
|
|
249
|
-
try {
|
|
250
|
-
await fs.promises.rename(path, old)
|
|
251
|
-
} catch (err) {
|
|
252
|
-
if (err.code !== 'ENOENT') throw err
|
|
253
|
-
}
|
|
254
|
-
await fs.promises.rename(tmp, path)
|
|
255
|
-
try {
|
|
256
|
-
await fs.promises.rm(old)
|
|
257
|
-
} catch (err) {
|
|
258
|
-
if (err.code !== 'ENOENT') throw err
|
|
259
|
-
}
|
|
260
|
-
}
|
package/dist/src/batch.d.ts
DELETED
|
@@ -1,62 +0,0 @@
|
|
|
1
|
-
export function create(blocks: API.BlockFetcher, root: API.ShardLink): Promise<API.Batcher>;
|
|
2
|
-
export type BatcherShardEntry = [
|
|
3
|
-
key: string,
|
|
4
|
-
value: API.ShardEntryValueValue | [API.ShardLink] | [API.ShardLink, Link.UnknownLink],
|
|
5
|
-
batcher?: Batcher
|
|
6
|
-
];
|
|
7
|
-
import * as API from './api.js';
|
|
8
|
-
/** @implements {API.Batcher} */
|
|
9
|
-
declare class Batcher implements API.Batcher {
|
|
10
|
-
/**
|
|
11
|
-
* @param {ShardFetcher} shards Shard storage.
|
|
12
|
-
* @param {API.ShardLink} link CID of the shard block.
|
|
13
|
-
* @param {string} prefix
|
|
14
|
-
*/
|
|
15
|
-
static create(shards: ShardFetcher, link: API.ShardLink, prefix: string): Promise<Batcher>;
|
|
16
|
-
/**
|
|
17
|
-
* @param {object} arg
|
|
18
|
-
* @param {ShardFetcher} arg.shards Shard storage.
|
|
19
|
-
* @param {BatcherShardEntry[]} arg.entries The entries in this shard.
|
|
20
|
-
* @param {string} arg.prefix Key prefix.
|
|
21
|
-
* @param {API.ShardConfig} arg.config Shard config.
|
|
22
|
-
* @param {API.ShardBlockView} [arg.base] Original shard this batcher is based on.
|
|
23
|
-
*/
|
|
24
|
-
constructor({ shards, entries, prefix, config, base }: {
|
|
25
|
-
shards: ShardFetcher;
|
|
26
|
-
entries: BatcherShardEntry[];
|
|
27
|
-
prefix: string;
|
|
28
|
-
config: API.ShardConfig;
|
|
29
|
-
base?: API.ShardBlockView | undefined;
|
|
30
|
-
});
|
|
31
|
-
shards: ShardFetcher;
|
|
32
|
-
prefix: string;
|
|
33
|
-
entries: BatcherShardEntry[];
|
|
34
|
-
config: API.ShardConfig;
|
|
35
|
-
base: API.ShardBlockView | undefined;
|
|
36
|
-
/**
|
|
37
|
-
* @param {string} key The key of the value to put.
|
|
38
|
-
* @param {API.UnknownLink} value The value to put.
|
|
39
|
-
* @returns {Promise<void>}
|
|
40
|
-
*/
|
|
41
|
-
put(key: string, value: API.UnknownLink): Promise<void>;
|
|
42
|
-
/**
|
|
43
|
-
* Traverse from this batcher through the shard to the correct batcher for
|
|
44
|
-
* the passed key.
|
|
45
|
-
*
|
|
46
|
-
* @param {string} key
|
|
47
|
-
* @returns {Promise<{ batcher: Batcher, key: string }>}
|
|
48
|
-
*/
|
|
49
|
-
traverse(key: string): Promise<{
|
|
50
|
-
batcher: Batcher;
|
|
51
|
-
key: string;
|
|
52
|
-
}>;
|
|
53
|
-
commit(): Promise<{
|
|
54
|
-
root: import("multiformats").CID<API.Shard, 113, 18, 1>;
|
|
55
|
-
additions: API.ShardBlockView[];
|
|
56
|
-
removals: API.ShardBlockView[];
|
|
57
|
-
}>;
|
|
58
|
-
}
|
|
59
|
-
import * as Link from 'multiformats/link';
|
|
60
|
-
import { ShardFetcher } from './shard.js';
|
|
61
|
-
export {};
|
|
62
|
-
//# sourceMappingURL=batch.d.ts.map
|
package/dist/src/batch.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../src/batch.js"],"names":[],"mappings":"AA2NO,+BAJI,IAAI,YAAY,QAChB,IAAI,SAAS,GACX,QAAQ,WAAW,CAAC,CAKhC;gCAvNY;IACZ,GAAO,EAAE,MAAM;IACf,KAAS,EAAE,IAAI,oBAAoB,sDAA4D;IAC/F,OAAW,CAAC,EAAE,OAAO;CAClB;qBATiB,UAAU;AAc/B,gCAAgC;AAChC,iCADiB,GAAG,CAAC,OAAO;IA4K1B;;;;OAIG;IACH,sBAJW,YAAY,QACZ,IAAI,SAAS,UACb,MAAM,oBAWhB;IAxLD;;;;;;;OAOG;IACH;QAN6B,MAAM,EAAxB,YAAY;QACa,OAAO,EAAhC,mBAAmB;QACP,MAAM,EAAlB,MAAM;QACe,MAAM,EAA3B,IAAI,WAAW;QACU,IAAI;OAQvC;IALC,qBAAoB;IACpB,eAAoB;IACpB,6BAAsB;IACtB,wBAAoB;IACpB,qCAAgB;IAGlB;;;;OAIG;IACH,SAJW,MAAM,SACN,IAAI,WAAW,GACb,QAAQ,IAAI,CAAC,CA6FzB;IAED;;;;;;OAMG;IACH,cAHW,MAAM,GACJ,QAAQ;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAetD;IAED;;;;OAgCC;CAiBF;sBA3MqB,mBAAmB;6BAGZ,YAAY"}
|
package/dist/src/crdt/batch.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../../src/crdt/batch.js"],"names":[],"mappings":"AASO,+BAJI,IAAI,YAAY,QAChB,IAAI,SAAS,GACX,QAAQ,IAAI,OAAO,CAAC,CAKhC;qBAXoB,WAAW"}
|
package/src/api.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {}
|
package/src/api.ts
DELETED
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats'
|
|
2
|
-
import { sha256 } from 'multiformats/hashes/sha2'
|
|
3
|
-
import * as dagCBOR from '@ipld/dag-cbor'
|
|
4
|
-
|
|
5
|
-
export { Link, UnknownLink, BlockView, Block, Version }
|
|
6
|
-
|
|
7
|
-
export type ShardEntryValueValue = UnknownLink
|
|
8
|
-
|
|
9
|
-
export type ShardEntryLinkValue = [ShardLink]
|
|
10
|
-
|
|
11
|
-
export type ShardEntryLinkAndValueValue = [ShardLink, UnknownLink]
|
|
12
|
-
|
|
13
|
-
export type ShardValueEntry = [key: string, value: ShardEntryValueValue]
|
|
14
|
-
|
|
15
|
-
export type ShardLinkEntry = [key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue]
|
|
16
|
-
|
|
17
|
-
/** Single key/value entry within a shard. */
|
|
18
|
-
export type ShardEntry = [key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue]
|
|
19
|
-
|
|
20
|
-
export interface Shard extends ShardConfig {
|
|
21
|
-
entries: ShardEntry[]
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export type ShardLink = Link<Shard, typeof dagCBOR.code, typeof sha256.code, 1>
|
|
25
|
-
|
|
26
|
-
export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {}
|
|
27
|
-
|
|
28
|
-
export interface ShardDiff {
|
|
29
|
-
additions: ShardBlockView[]
|
|
30
|
-
removals: ShardBlockView[]
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export interface BlockFetcher {
|
|
34
|
-
get<T = unknown, C extends number = number, A extends number = number, V extends Version = 1> (link: Link<T, C, A, V>):
|
|
35
|
-
Promise<Block<T, C, A, V> | undefined>
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
export interface ShardConfig {
|
|
39
|
-
/** Shard compatibility version. */
|
|
40
|
-
version: number
|
|
41
|
-
/**
|
|
42
|
-
* Characters allowed in keys, referring to a known character set.
|
|
43
|
-
* e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
|
|
44
|
-
*/
|
|
45
|
-
keyChars: string
|
|
46
|
-
/** Max key size in bytes - default 4096 bytes. */
|
|
47
|
-
maxKeySize: number
|
|
48
|
-
/** The key prefix from the root to this shard. */
|
|
49
|
-
prefix: string
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
export type ShardOptions = Partial<ShardConfig>
|
|
53
|
-
|
|
54
|
-
export interface KeyPrefixOption {
|
|
55
|
-
/** Filter results to entries with keys prefixed with this string. */
|
|
56
|
-
prefix: string
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
export type KeyRangeOption =
|
|
60
|
-
| KeyLowerBoundRangeOption
|
|
61
|
-
| KeyUpperBoundRangeOption
|
|
62
|
-
| (KeyLowerBoundRangeOption & KeyUpperBoundRangeOption)
|
|
63
|
-
|
|
64
|
-
export type KeyLowerBoundRangeOption =
|
|
65
|
-
| KeyLowerBoundRangeExclusiveOption
|
|
66
|
-
| KeyLowerBoundRangeInclusiveOption
|
|
67
|
-
|
|
68
|
-
export interface KeyLowerBoundRangeExclusiveOption {
|
|
69
|
-
gt: string
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
export interface KeyLowerBoundRangeInclusiveOption {
|
|
73
|
-
gte: string
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
export type KeyUpperBoundRangeOption =
|
|
77
|
-
| KeyUpperBoundRangeExclusiveOption
|
|
78
|
-
| KeyUpperBoundRangeInclusiveOption
|
|
79
|
-
|
|
80
|
-
export interface KeyUpperBoundRangeExclusiveOption {
|
|
81
|
-
lt: string
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
export interface KeyUpperBoundRangeInclusiveOption {
|
|
85
|
-
lte: string
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
export type EntriesOptions =
|
|
89
|
-
| KeyPrefixOption
|
|
90
|
-
| KeyRangeOption
|
package/src/batch/api.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {}
|