@web3-storage/pail 0.6.2 → 0.6.3-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/dist/cli.d.ts +0 -3
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -208
- package/dist/scripts/randomcid.d.ts +0 -2
- package/dist/scripts/randomcid.d.ts.map +0 -1
- package/dist/scripts/randomcid.js +0 -10
- package/dist/scripts/words/gen.d.ts +0 -2
- package/dist/scripts/words/gen.d.ts.map +0 -1
- package/dist/scripts/words/gen.js +0 -51
- package/dist/src/api.d.ts +0 -61
- package/dist/src/api.d.ts.map +0 -1
- package/dist/src/api.js +0 -1
- package/dist/src/batch/api.d.ts +0 -31
- package/dist/src/batch/api.d.ts.map +0 -1
- package/dist/src/batch/api.js +0 -1
- package/dist/src/batch/index.d.ts +0 -23
- package/dist/src/batch/index.d.ts.map +0 -1
- package/dist/src/batch/index.js +0 -241
- package/dist/src/batch/shard.d.ts +0 -3
- package/dist/src/batch/shard.d.ts.map +0 -1
- package/dist/src/batch/shard.js +0 -12
- package/dist/src/block.d.ts +0 -35
- package/dist/src/block.d.ts.map +0 -1
- package/dist/src/block.js +0 -66
- package/dist/src/clock/api.d.ts +0 -10
- package/dist/src/clock/api.d.ts.map +0 -1
- package/dist/src/clock/api.js +0 -1
- package/dist/src/clock/index.d.ts +0 -48
- package/dist/src/clock/index.d.ts.map +0 -1
- package/dist/src/clock/index.js +0 -199
- package/dist/src/crdt/api.d.ts +0 -26
- package/dist/src/crdt/api.d.ts.map +0 -1
- package/dist/src/crdt/api.js +0 -1
- package/dist/src/crdt/batch/api.d.ts +0 -11
- package/dist/src/crdt/batch/api.d.ts.map +0 -1
- package/dist/src/crdt/batch/api.js +0 -1
- package/dist/src/crdt/batch/index.d.ts +0 -5
- package/dist/src/crdt/batch/index.d.ts.map +0 -1
- package/dist/src/crdt/batch/index.js +0 -140
- package/dist/src/crdt/index.d.ts +0 -9
- package/dist/src/crdt/index.d.ts.map +0 -1
- package/dist/src/crdt/index.js +0 -344
- package/dist/src/diff.d.ts +0 -13
- package/dist/src/diff.d.ts.map +0 -1
- package/dist/src/diff.js +0 -151
- package/dist/src/index.d.ts +0 -10
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js +0 -356
- package/dist/src/merge.d.ts +0 -5
- package/dist/src/merge.d.ts.map +0 -1
- package/dist/src/merge.js +0 -42
- package/dist/src/shard.d.ts +0 -43
- package/dist/src/shard.d.ts.map +0 -1
- package/dist/src/shard.js +0 -166
- package/dist/test/batch.test.d.ts +0 -2
- package/dist/test/batch.test.d.ts.map +0 -1
- package/dist/test/batch.test.js +0 -126
- package/dist/test/clock.test.d.ts +0 -2
- package/dist/test/clock.test.d.ts.map +0 -1
- package/dist/test/clock.test.js +0 -244
- package/dist/test/crdt.test.d.ts +0 -2
- package/dist/test/crdt.test.d.ts.map +0 -1
- package/dist/test/crdt.test.js +0 -271
- package/dist/test/del.test.d.ts +0 -2
- package/dist/test/del.test.d.ts.map +0 -1
- package/dist/test/del.test.js +0 -21
- package/dist/test/diff.test.d.ts +0 -2
- package/dist/test/diff.test.d.ts.map +0 -1
- package/dist/test/diff.test.js +0 -38
- package/dist/test/entries.test.d.ts +0 -2
- package/dist/test/entries.test.d.ts.map +0 -1
- package/dist/test/entries.test.js +0 -210
- package/dist/test/get.test.d.ts +0 -2
- package/dist/test/get.test.d.ts.map +0 -1
- package/dist/test/get.test.js +0 -26
- package/dist/test/helpers.d.ts +0 -35
- package/dist/test/helpers.d.ts.map +0 -1
- package/dist/test/helpers.js +0 -168
- package/dist/test/put.test.d.ts +0 -2
- package/dist/test/put.test.d.ts.map +0 -1
- package/dist/test/put.test.js +0 -165
- package/dist/tsconfig.tsbuildinfo +0 -1
- package/dist/vitest.config.d.ts +0 -3
- package/dist/vitest.config.d.ts.map +0 -1
- package/dist/vitest.config.js +0 -13
package/dist/src/diff.js
DELETED
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
// eslint-disable-next-line no-unused-vars
|
|
2
|
-
import * as API from './api.js';
|
|
3
|
-
import { ShardFetcher } from './shard.js';
|
|
4
|
-
/**
|
|
5
|
-
* @typedef {string} K
|
|
6
|
-
* @typedef {[before: null, after: API.UnknownLink]} AddV
|
|
7
|
-
* @typedef {[before: API.UnknownLink, after: API.UnknownLink]} UpdateV
|
|
8
|
-
* @typedef {[before: API.UnknownLink, after: null]} DeleteV
|
|
9
|
-
* @typedef {[key: K, value: AddV|UpdateV|DeleteV]} KV
|
|
10
|
-
* @typedef {KV[]} KeysDiff
|
|
11
|
-
* @typedef {{ keys: KeysDiff, shards: API.ShardDiff }} CombinedDiff
|
|
12
|
-
*/
|
|
13
|
-
/**
|
|
14
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
15
|
-
* @param {API.ShardLink} a Base DAG.
|
|
16
|
-
* @param {API.ShardLink} b Comparison DAG.
|
|
17
|
-
* @returns {Promise<CombinedDiff>}
|
|
18
|
-
*/
|
|
19
|
-
export const difference = async (blocks, a, b) => {
|
|
20
|
-
if (isEqual(a, b))
|
|
21
|
-
return { keys: [], shards: { additions: [], removals: [] } };
|
|
22
|
-
const shards = new ShardFetcher(blocks);
|
|
23
|
-
const [ashard, bshard] = await Promise.all([shards.get(a), shards.get(b)]);
|
|
24
|
-
const aents = new Map(ashard.value.entries);
|
|
25
|
-
const bents = new Map(bshard.value.entries);
|
|
26
|
-
const keys = /** @type {Map<K, AddV|UpdateV|DeleteV>} */ (new Map());
|
|
27
|
-
const additions = new Map([[bshard.cid.toString(), bshard]]);
|
|
28
|
-
const removals = new Map([[ashard.cid.toString(), ashard]]);
|
|
29
|
-
// find shards removed in B
|
|
30
|
-
for (const [akey, aval] of ashard.value.entries) {
|
|
31
|
-
const bval = bents.get(akey);
|
|
32
|
-
if (bval)
|
|
33
|
-
continue;
|
|
34
|
-
if (!Array.isArray(aval)) {
|
|
35
|
-
keys.set(`${ashard.value.prefix}${akey}`, [aval, null]);
|
|
36
|
-
continue;
|
|
37
|
-
}
|
|
38
|
-
// if shard link _with_ value
|
|
39
|
-
if (aval[1] != null) {
|
|
40
|
-
keys.set(`${ashard.value.prefix}${akey}`, [aval[1], null]);
|
|
41
|
-
}
|
|
42
|
-
for await (const s of collect(shards, aval[0])) {
|
|
43
|
-
for (const [k, v] of s.value.entries) {
|
|
44
|
-
if (!Array.isArray(v)) {
|
|
45
|
-
keys.set(`${s.value.prefix}${k}`, [v, null]);
|
|
46
|
-
}
|
|
47
|
-
else if (v[1] != null) {
|
|
48
|
-
keys.set(`${s.value.prefix}${k}`, [v[1], null]);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
removals.set(s.cid.toString(), s);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
// find shards added or updated in B
|
|
55
|
-
for (const [bkey, bval] of bshard.value.entries) {
|
|
56
|
-
const aval = aents.get(bkey);
|
|
57
|
-
if (!Array.isArray(bval)) {
|
|
58
|
-
if (!aval) {
|
|
59
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [null, bval]);
|
|
60
|
-
}
|
|
61
|
-
else if (Array.isArray(aval)) {
|
|
62
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval]);
|
|
63
|
-
}
|
|
64
|
-
else if (!isEqual(aval, bval)) {
|
|
65
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval]);
|
|
66
|
-
}
|
|
67
|
-
continue;
|
|
68
|
-
}
|
|
69
|
-
if (aval && Array.isArray(aval)) { // updated in B
|
|
70
|
-
if (isEqual(aval[0], bval[0])) {
|
|
71
|
-
if (bval[1] != null && (aval[1] == null || !isEqual(aval[1], bval[1]))) {
|
|
72
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [aval[1] ?? null, bval[1]]);
|
|
73
|
-
}
|
|
74
|
-
continue; // updated value?
|
|
75
|
-
}
|
|
76
|
-
const res = await difference(blocks, aval[0], bval[0]);
|
|
77
|
-
for (const shard of res.shards.additions) {
|
|
78
|
-
additions.set(shard.cid.toString(), shard);
|
|
79
|
-
}
|
|
80
|
-
for (const shard of res.shards.removals) {
|
|
81
|
-
removals.set(shard.cid.toString(), shard);
|
|
82
|
-
}
|
|
83
|
-
for (const [k, v] of res.keys) {
|
|
84
|
-
keys.set(k, v);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
else if (aval) { // updated in B value => link+value
|
|
88
|
-
if (bval[1] == null) {
|
|
89
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [aval, null]);
|
|
90
|
-
}
|
|
91
|
-
else if (!isEqual(aval, bval[1])) {
|
|
92
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [aval, bval[1]]);
|
|
93
|
-
}
|
|
94
|
-
for await (const s of collect(shards, bval[0])) {
|
|
95
|
-
for (const [k, v] of s.value.entries) {
|
|
96
|
-
if (!Array.isArray(v)) {
|
|
97
|
-
keys.set(`${s.value.prefix}${k}`, [null, v]);
|
|
98
|
-
}
|
|
99
|
-
else if (v[1] != null) {
|
|
100
|
-
keys.set(`${s.value.prefix}${k}`, [null, v[1]]);
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
additions.set(s.cid.toString(), s);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
else { // added in B
|
|
107
|
-
keys.set(`${bshard.value.prefix}${bkey}`, [null, bval[0]]);
|
|
108
|
-
for await (const s of collect(shards, bval[0])) {
|
|
109
|
-
for (const [k, v] of s.value.entries) {
|
|
110
|
-
if (!Array.isArray(v)) {
|
|
111
|
-
keys.set(`${s.value.prefix}${k}`, [null, v]);
|
|
112
|
-
}
|
|
113
|
-
else if (v[1] != null) {
|
|
114
|
-
keys.set(`${s.value.prefix}${k}`, [null, v[1]]);
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
additions.set(s.cid.toString(), s);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
// filter blocks that were added _and_ removed from B
|
|
122
|
-
for (const k of removals.keys()) {
|
|
123
|
-
if (additions.has(k)) {
|
|
124
|
-
additions.delete(k);
|
|
125
|
-
removals.delete(k);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
return {
|
|
129
|
-
keys: [...keys.entries()].sort((a, b) => a[0] < b[0] ? -1 : 1),
|
|
130
|
-
shards: { additions: [...additions.values()], removals: [...removals.values()] }
|
|
131
|
-
};
|
|
132
|
-
};
|
|
133
|
-
/**
|
|
134
|
-
* @param {API.UnknownLink} a
|
|
135
|
-
* @param {API.UnknownLink} b
|
|
136
|
-
*/
|
|
137
|
-
const isEqual = (a, b) => a.toString() === b.toString();
|
|
138
|
-
/**
|
|
139
|
-
* @param {import('./shard.js').ShardFetcher} shards
|
|
140
|
-
* @param {API.ShardLink} root
|
|
141
|
-
* @returns {AsyncIterableIterator<API.ShardBlockView>}
|
|
142
|
-
*/
|
|
143
|
-
async function* collect(shards, root) {
|
|
144
|
-
const shard = await shards.get(root);
|
|
145
|
-
yield shard;
|
|
146
|
-
for (const [, v] of shard.value.entries) {
|
|
147
|
-
if (!Array.isArray(v))
|
|
148
|
-
continue;
|
|
149
|
-
yield* collect(shards, v[0]);
|
|
150
|
-
}
|
|
151
|
-
}
|
package/dist/src/index.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export function put(blocks: API.BlockFetcher, root: API.ShardLink, key: string, value: API.UnknownLink): Promise<{
|
|
2
|
-
root: API.ShardLink;
|
|
3
|
-
} & API.ShardDiff>;
|
|
4
|
-
export function get(blocks: API.BlockFetcher, root: API.ShardLink, key: string): Promise<API.UnknownLink | undefined>;
|
|
5
|
-
export function del(blocks: API.BlockFetcher, root: API.ShardLink, key: string): Promise<{
|
|
6
|
-
root: API.ShardLink;
|
|
7
|
-
} & API.ShardDiff>;
|
|
8
|
-
export function entries(blocks: API.BlockFetcher, root: API.ShardLink, options?: API.EntriesOptions): AsyncIterableIterator<API.ShardValueEntry>;
|
|
9
|
-
import * as API from './api.js';
|
|
10
|
-
//# sourceMappingURL=index.d.ts.map
|
package/dist/src/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":"AAeO,4BANI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,OACb,MAAM,SACN,GAAG,CAAC,WAAW,GACb,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAA;CAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAkI5D;AAWM,4BALI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,OACb,MAAM,GACJ,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,SAAS,CAAC,CAWhD;AAWM,4BALI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,OACb,MAAM,GACJ,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAA;CAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAoE5D;AAgEM,gCALI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,YACb,GAAG,CAAC,cAAc,GAChB,qBAAqB,CAAC,GAAG,CAAC,eAAe,CAAC,CA4EtD;qBAzXoB,UAAU"}
|
package/dist/src/index.js
DELETED
|
@@ -1,356 +0,0 @@
|
|
|
1
|
-
// eslint-disable-next-line no-unused-vars
|
|
2
|
-
import * as API from './api.js';
|
|
3
|
-
import { ShardFetcher, isPrintableASCII } from './shard.js';
|
|
4
|
-
import * as Shard from './shard.js';
|
|
5
|
-
/**
|
|
6
|
-
* Put a value (a CID) for the given key. If the key exists it's value is
|
|
7
|
-
* overwritten.
|
|
8
|
-
*
|
|
9
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
10
|
-
* @param {API.ShardLink} root CID of the root node of the bucket.
|
|
11
|
-
* @param {string} key The key of the value to put.
|
|
12
|
-
* @param {API.UnknownLink} value The value to put.
|
|
13
|
-
* @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
|
|
14
|
-
*/
|
|
15
|
-
export const put = async (blocks, root, key, value) => {
|
|
16
|
-
const shards = new ShardFetcher(blocks);
|
|
17
|
-
const rshard = await shards.get(root);
|
|
18
|
-
if (rshard.value.keyChars !== Shard.KeyCharsASCII) {
|
|
19
|
-
throw new Error(`unsupported key character set: ${rshard.value.keyChars}`);
|
|
20
|
-
}
|
|
21
|
-
if (!isPrintableASCII(key)) {
|
|
22
|
-
throw new Error('key contains non-ASCII characters');
|
|
23
|
-
}
|
|
24
|
-
// ensure utf8 encoded key is smaller than max
|
|
25
|
-
if (new TextEncoder().encode(key).length > rshard.value.maxKeySize) {
|
|
26
|
-
throw new Error(`UTF-8 encoded key exceeds max size of ${rshard.value.maxKeySize} bytes`);
|
|
27
|
-
}
|
|
28
|
-
const path = await traverse(shards, rshard, key);
|
|
29
|
-
const target = path[path.length - 1];
|
|
30
|
-
const skey = key.slice(target.value.prefix.length); // key within the shard
|
|
31
|
-
/** @type {API.ShardEntry} */
|
|
32
|
-
let entry = [skey, value];
|
|
33
|
-
const targetEntries = [...target.value.entries];
|
|
34
|
-
/** @type {API.ShardBlockView[]} */
|
|
35
|
-
const additions = [];
|
|
36
|
-
for (const [i, e] of targetEntries.entries()) {
|
|
37
|
-
const [k, v] = e;
|
|
38
|
-
// is this just a replace?
|
|
39
|
-
if (k === skey)
|
|
40
|
-
break;
|
|
41
|
-
// do we need to shard this entry?
|
|
42
|
-
const shortest = k.length < skey.length ? k : skey;
|
|
43
|
-
const other = shortest === k ? skey : k;
|
|
44
|
-
let common = '';
|
|
45
|
-
for (const char of shortest) {
|
|
46
|
-
const next = common + char;
|
|
47
|
-
if (!other.startsWith(next))
|
|
48
|
-
break;
|
|
49
|
-
common = next;
|
|
50
|
-
}
|
|
51
|
-
if (common.length) {
|
|
52
|
-
/** @type {API.ShardEntry[]} */
|
|
53
|
-
let entries = [];
|
|
54
|
-
// if the existing entry key or new key is equal to the common prefix,
|
|
55
|
-
// then the existing value / new value needs to persist in the parent
|
|
56
|
-
// shard. Otherwise they persist in this new shard.
|
|
57
|
-
if (common !== skey) {
|
|
58
|
-
entries = Shard.putEntry(entries, [skey.slice(common.length), value]);
|
|
59
|
-
}
|
|
60
|
-
if (common !== k) {
|
|
61
|
-
entries = Shard.putEntry(entries, [k.slice(common.length), v]);
|
|
62
|
-
}
|
|
63
|
-
let child = await Shard.encodeBlock(Shard.withEntries(entries, { ...target.value, prefix: target.value.prefix + common }));
|
|
64
|
-
additions.push(child);
|
|
65
|
-
// need to spread as access by index does not consider utf-16 surrogates
|
|
66
|
-
const commonChars = [...common];
|
|
67
|
-
// create parent shards for each character of the common prefix
|
|
68
|
-
for (let i = commonChars.length - 1; i > 0; i--) {
|
|
69
|
-
const parentConfig = { ...target.value, prefix: target.value.prefix + commonChars.slice(0, i).join('') };
|
|
70
|
-
/** @type {API.ShardEntryLinkValue | API.ShardEntryValueValue | API.ShardEntryLinkAndValueValue} */
|
|
71
|
-
let parentValue;
|
|
72
|
-
// if the first iteration and the existing entry key is equal to the
|
|
73
|
-
// common prefix, then existing value needs to persist in this parent
|
|
74
|
-
if (i === commonChars.length - 1 && common === k) {
|
|
75
|
-
if (Array.isArray(v))
|
|
76
|
-
throw new Error('found a shard link when expecting a value');
|
|
77
|
-
parentValue = [child.cid, v];
|
|
78
|
-
}
|
|
79
|
-
else if (i === commonChars.length - 1 && common === skey) {
|
|
80
|
-
parentValue = [child.cid, value];
|
|
81
|
-
}
|
|
82
|
-
else {
|
|
83
|
-
parentValue = [child.cid];
|
|
84
|
-
}
|
|
85
|
-
const parent = await Shard.encodeBlock(Shard.withEntries([[commonChars[i], parentValue]], parentConfig));
|
|
86
|
-
additions.push(parent);
|
|
87
|
-
child = parent;
|
|
88
|
-
}
|
|
89
|
-
// remove the sharded entry
|
|
90
|
-
targetEntries.splice(i, 1);
|
|
91
|
-
// create the entry that will be added to target
|
|
92
|
-
if (commonChars.length === 1 && common === k) {
|
|
93
|
-
if (Array.isArray(v))
|
|
94
|
-
throw new Error('found a shard link when expecting a value');
|
|
95
|
-
entry = [commonChars[0], [child.cid, v]];
|
|
96
|
-
}
|
|
97
|
-
else if (commonChars.length === 1 && common === skey) {
|
|
98
|
-
entry = [commonChars[0], [child.cid, value]];
|
|
99
|
-
}
|
|
100
|
-
else {
|
|
101
|
-
entry = [commonChars[0], [child.cid]];
|
|
102
|
-
}
|
|
103
|
-
break;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
const shard = Shard.withEntries(Shard.putEntry(targetEntries, entry), target.value);
|
|
107
|
-
let child = await Shard.encodeBlock(shard);
|
|
108
|
-
// if no change in the target then we're done
|
|
109
|
-
if (child.cid.toString() === target.cid.toString()) {
|
|
110
|
-
return { root, additions: [], removals: [] };
|
|
111
|
-
}
|
|
112
|
-
additions.push(child);
|
|
113
|
-
// path is root -> target, so work backwards, propagating the new shard CID
|
|
114
|
-
for (let i = path.length - 2; i >= 0; i--) {
|
|
115
|
-
const parent = path[i];
|
|
116
|
-
const key = child.value.prefix.slice(parent.value.prefix.length);
|
|
117
|
-
const value = Shard.withEntries(parent.value.entries.map((entry) => {
|
|
118
|
-
const [k, v] = entry;
|
|
119
|
-
if (k !== key)
|
|
120
|
-
return entry;
|
|
121
|
-
if (!Array.isArray(v))
|
|
122
|
-
throw new Error(`"${key}" is not a shard link in: ${parent.cid}`);
|
|
123
|
-
return /** @type {API.ShardEntry} */ (v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]);
|
|
124
|
-
}), parent.value);
|
|
125
|
-
child = await Shard.encodeBlock(value);
|
|
126
|
-
additions.push(child);
|
|
127
|
-
}
|
|
128
|
-
return { root: additions[additions.length - 1].cid, additions, removals: path };
|
|
129
|
-
};
|
|
130
|
-
/**
|
|
131
|
-
* Get the stored value for the given key from the bucket. If the key is not
|
|
132
|
-
* found, `undefined` is returned.
|
|
133
|
-
*
|
|
134
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
135
|
-
* @param {API.ShardLink} root CID of the root node of the bucket.
|
|
136
|
-
* @param {string} key The key of the value to get.
|
|
137
|
-
* @returns {Promise<API.UnknownLink | undefined>}
|
|
138
|
-
*/
|
|
139
|
-
export const get = async (blocks, root, key) => {
|
|
140
|
-
const shards = new ShardFetcher(blocks);
|
|
141
|
-
const rshard = await shards.get(root);
|
|
142
|
-
const path = await traverse(shards, rshard, key);
|
|
143
|
-
const target = path[path.length - 1];
|
|
144
|
-
const skey = key.slice(target.value.prefix.length); // key within the shard
|
|
145
|
-
const entry = target.value.entries.find(([k]) => k === skey);
|
|
146
|
-
if (!entry)
|
|
147
|
-
return;
|
|
148
|
-
return Array.isArray(entry[1]) ? entry[1][1] : entry[1];
|
|
149
|
-
};
|
|
150
|
-
/**
|
|
151
|
-
* Delete the value for the given key from the bucket. If the key is not found
|
|
152
|
-
* no operation occurs.
|
|
153
|
-
*
|
|
154
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
155
|
-
* @param {API.ShardLink} root CID of the root node of the bucket.
|
|
156
|
-
* @param {string} key The key of the value to delete.
|
|
157
|
-
* @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
|
|
158
|
-
*/
|
|
159
|
-
export const del = async (blocks, root, key) => {
|
|
160
|
-
const shards = new ShardFetcher(blocks);
|
|
161
|
-
const rshard = await shards.get(root);
|
|
162
|
-
const path = await traverse(shards, rshard, key);
|
|
163
|
-
const target = path[path.length - 1];
|
|
164
|
-
const skey = key.slice(target.value.prefix.length); // key within the shard
|
|
165
|
-
const entryidx = target.value.entries.findIndex(([k]) => k === skey);
|
|
166
|
-
if (entryidx === -1)
|
|
167
|
-
return { root, additions: [], removals: [] };
|
|
168
|
-
const entry = target.value.entries[entryidx];
|
|
169
|
-
// cannot delete a shard (without data)
|
|
170
|
-
if (Array.isArray(entry[1]) && entry[1][1] == null) {
|
|
171
|
-
return { root, additions: [], removals: [] };
|
|
172
|
-
}
|
|
173
|
-
/** @type {API.ShardBlockView[]} */
|
|
174
|
-
const additions = [];
|
|
175
|
-
/** @type {API.ShardBlockView[]} */
|
|
176
|
-
const removals = [...path];
|
|
177
|
-
let shard = Shard.withEntries([...target.value.entries], target.value);
|
|
178
|
-
if (Array.isArray(entry[1])) {
|
|
179
|
-
// remove the value from this link+value
|
|
180
|
-
shard.entries[entryidx] = [entry[0], [entry[1][0]]];
|
|
181
|
-
}
|
|
182
|
-
else {
|
|
183
|
-
shard.entries.splice(entryidx, 1);
|
|
184
|
-
// if now empty, remove from parent
|
|
185
|
-
while (!shard.entries.length) {
|
|
186
|
-
const child = path[path.length - 1];
|
|
187
|
-
const parent = path[path.length - 2];
|
|
188
|
-
if (!parent)
|
|
189
|
-
break;
|
|
190
|
-
path.pop();
|
|
191
|
-
shard = Shard.withEntries(parent.value.entries.filter(e => {
|
|
192
|
-
if (!Array.isArray(e[1]))
|
|
193
|
-
return true;
|
|
194
|
-
return e[1][0].toString() !== child.cid.toString();
|
|
195
|
-
}), parent.value);
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
let child = await Shard.encodeBlock(shard);
|
|
199
|
-
additions.push(child);
|
|
200
|
-
// path is root -> shard, so work backwards, propagating the new shard CID
|
|
201
|
-
for (let i = path.length - 2; i >= 0; i--) {
|
|
202
|
-
const parent = path[i];
|
|
203
|
-
const key = child.value.prefix.slice(parent.value.prefix.length);
|
|
204
|
-
const value = Shard.withEntries(parent.value.entries.map((entry) => {
|
|
205
|
-
const [k, v] = entry;
|
|
206
|
-
if (k !== key)
|
|
207
|
-
return entry;
|
|
208
|
-
if (!Array.isArray(v))
|
|
209
|
-
throw new Error(`"${key}" is not a shard link in: ${parent.cid}`);
|
|
210
|
-
return /** @type {API.ShardEntry} */ (v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]);
|
|
211
|
-
}), parent.value);
|
|
212
|
-
child = await Shard.encodeBlock(value);
|
|
213
|
-
additions.push(child);
|
|
214
|
-
}
|
|
215
|
-
return { root: additions[additions.length - 1].cid, additions, removals };
|
|
216
|
-
};
|
|
217
|
-
/**
|
|
218
|
-
* @param {API.EntriesOptions} [options]
|
|
219
|
-
* @returns {options is API.KeyPrefixOption}
|
|
220
|
-
*/
|
|
221
|
-
const isKeyPrefixOption = options => {
|
|
222
|
-
const opts = options ?? {};
|
|
223
|
-
return 'prefix' in opts && Boolean(opts.prefix);
|
|
224
|
-
};
|
|
225
|
-
/**
|
|
226
|
-
* @param {API.EntriesOptions} [options]
|
|
227
|
-
* @returns {options is API.KeyRangeOption}
|
|
228
|
-
*/
|
|
229
|
-
const isKeyRangeOption = options => {
|
|
230
|
-
const opts = options ?? {};
|
|
231
|
-
return ('gt' in opts && Boolean(opts.gt)) || ('gte' in opts && Boolean(opts.gte)) || ('lt' in opts && Boolean(opts.lt)) || ('lte' in opts && Boolean(opts.lte));
|
|
232
|
-
};
|
|
233
|
-
/**
|
|
234
|
-
* @param {API.KeyRangeOption} options
|
|
235
|
-
* @returns {options is API.KeyLowerBoundRangeOption}
|
|
236
|
-
*/
|
|
237
|
-
const isKeyLowerBoundRangeOption = options => ('gt' in options && Boolean(options.gt)) || ('gte' in options && Boolean(options.gte));
|
|
238
|
-
/**
|
|
239
|
-
* @param {API.KeyLowerBoundRangeOption} options
|
|
240
|
-
* @returns {options is API.KeyLowerBoundRangeInclusiveOption}
|
|
241
|
-
*/
|
|
242
|
-
const isKeyLowerBoundRangeInclusiveOption = options => 'gte' in options && Boolean(options.gte);
|
|
243
|
-
/**
|
|
244
|
-
* @param {API.KeyLowerBoundRangeOption} options
|
|
245
|
-
* @returns {options is API.KeyLowerBoundRangeExclusiveOption}
|
|
246
|
-
*/
|
|
247
|
-
const isKeyLowerBoundRangeExclusiveOption = options => 'gt' in options && Boolean(options.gt);
|
|
248
|
-
/**
|
|
249
|
-
* @param {API.KeyRangeOption} options
|
|
250
|
-
* @returns {options is API.KeyUpperBoundRangeOption}
|
|
251
|
-
*/
|
|
252
|
-
const isKeyUpperBoundRangeOption = options => ('lt' in options && Boolean(options.lt)) || ('lte' in options && Boolean(options.lte));
|
|
253
|
-
/**
|
|
254
|
-
* @param {API.KeyUpperBoundRangeOption} options
|
|
255
|
-
* @returns {options is API.KeyUpperBoundRangeInclusiveOption}
|
|
256
|
-
*/
|
|
257
|
-
const isKeyUpperBoundRangeInclusiveOption = options => 'lte' in options && Boolean(options.lte);
|
|
258
|
-
/**
|
|
259
|
-
* @param {API.KeyUpperBoundRangeOption} options
|
|
260
|
-
* @returns {options is API.KeyUpperBoundRangeExclusiveOption}
|
|
261
|
-
*/
|
|
262
|
-
const isKeyUpperBoundRangeExclusiveOption = options => 'lt' in options && Boolean(options.lt);
|
|
263
|
-
/**
|
|
264
|
-
* List entries in the bucket.
|
|
265
|
-
*
|
|
266
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
267
|
-
* @param {API.ShardLink} root CID of the root node of the bucket.
|
|
268
|
-
* @param {API.EntriesOptions} [options]
|
|
269
|
-
* @returns {AsyncIterableIterator<API.ShardValueEntry>}
|
|
270
|
-
*/
|
|
271
|
-
export const entries = async function* (blocks, root, options) {
|
|
272
|
-
const hasKeyPrefix = isKeyPrefixOption(options);
|
|
273
|
-
const hasKeyRange = isKeyRangeOption(options);
|
|
274
|
-
const hasKeyLowerBoundRange = hasKeyRange && isKeyLowerBoundRangeOption(options);
|
|
275
|
-
const hasKeyLowerBoundRangeInclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeInclusiveOption(options);
|
|
276
|
-
const hasKeyLowerBoundRangeExclusive = hasKeyLowerBoundRange && isKeyLowerBoundRangeExclusiveOption(options);
|
|
277
|
-
const hasKeyUpperBoundRange = hasKeyRange && isKeyUpperBoundRangeOption(options);
|
|
278
|
-
const hasKeyUpperBoundRangeInclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeInclusiveOption(options);
|
|
279
|
-
const hasKeyUpperBoundRangeExclusive = hasKeyUpperBoundRange && isKeyUpperBoundRangeExclusiveOption(options);
|
|
280
|
-
const hasKeyUpperAndLowerBoundRange = hasKeyLowerBoundRange && hasKeyUpperBoundRange;
|
|
281
|
-
const shards = new ShardFetcher(blocks);
|
|
282
|
-
const rshard = await shards.get(root);
|
|
283
|
-
yield* (
|
|
284
|
-
/** @returns {AsyncIterableIterator<API.ShardValueEntry>} */
|
|
285
|
-
async function* ents(shard) {
|
|
286
|
-
for (const entry of shard.value.entries) {
|
|
287
|
-
const key = shard.value.prefix + entry[0];
|
|
288
|
-
// if array, this is a link to a shard
|
|
289
|
-
if (Array.isArray(entry[1])) {
|
|
290
|
-
if (entry[1][1]) {
|
|
291
|
-
if ((hasKeyPrefix && key.startsWith(options.prefix)) ||
|
|
292
|
-
(hasKeyUpperAndLowerBoundRange && (((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
|
|
293
|
-
((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)))) ||
|
|
294
|
-
(hasKeyLowerBoundRangeExclusive && key > options.gt) ||
|
|
295
|
-
(hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
|
|
296
|
-
(hasKeyUpperBoundRangeExclusive && key < options.lt) ||
|
|
297
|
-
(hasKeyUpperBoundRangeInclusive && key <= options.lte) ||
|
|
298
|
-
(!hasKeyPrefix && !hasKeyRange)) {
|
|
299
|
-
yield [key, entry[1][1]];
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
if (hasKeyPrefix) {
|
|
303
|
-
if (options.prefix.length <= key.length && !key.startsWith(options.prefix)) {
|
|
304
|
-
continue;
|
|
305
|
-
}
|
|
306
|
-
if (options.prefix.length > key.length && !options.prefix.startsWith(key)) {
|
|
307
|
-
continue;
|
|
308
|
-
}
|
|
309
|
-
}
|
|
310
|
-
else if ((hasKeyLowerBoundRangeExclusive && (trunc(key, Math.min(key.length, options.gt.length)) < trunc(options.gt, Math.min(key.length, options.gt.length)))) ||
|
|
311
|
-
(hasKeyLowerBoundRangeInclusive && (trunc(key, Math.min(key.length, options.gte.length)) < trunc(options.gte, Math.min(key.length, options.gte.length)))) ||
|
|
312
|
-
(hasKeyUpperBoundRangeExclusive && (trunc(key, Math.min(key.length, options.lt.length)) > trunc(options.lt, Math.min(key.length, options.lt.length)))) ||
|
|
313
|
-
(hasKeyUpperBoundRangeInclusive && (trunc(key, Math.min(key.length, options.lte.length)) > trunc(options.lte, Math.min(key.length, options.lte.length))))) {
|
|
314
|
-
continue;
|
|
315
|
-
}
|
|
316
|
-
yield* ents(await shards.get(entry[1][0]));
|
|
317
|
-
}
|
|
318
|
-
else {
|
|
319
|
-
if ((hasKeyPrefix && key.startsWith(options.prefix)) ||
|
|
320
|
-
(hasKeyRange && hasKeyUpperAndLowerBoundRange && (((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte)) &&
|
|
321
|
-
((hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte)))) ||
|
|
322
|
-
(hasKeyRange && !hasKeyUpperAndLowerBoundRange && ((hasKeyLowerBoundRangeExclusive && key > options.gt) || (hasKeyLowerBoundRangeInclusive && key >= options.gte) ||
|
|
323
|
-
(hasKeyUpperBoundRangeExclusive && key < options.lt) || (hasKeyUpperBoundRangeInclusive && key <= options.lte))) ||
|
|
324
|
-
(!hasKeyPrefix && !hasKeyRange)) {
|
|
325
|
-
yield [key, entry[1]];
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
}
|
|
329
|
-
})(rshard);
|
|
330
|
-
};
|
|
331
|
-
/**
|
|
332
|
-
* @param {string} str
|
|
333
|
-
* @param {number} len
|
|
334
|
-
*/
|
|
335
|
-
const trunc = (str, len) => str.length <= len ? str : str.slice(0, len);
|
|
336
|
-
/**
|
|
337
|
-
* Traverse from the passed shard block to the target shard block using the
|
|
338
|
-
* passed key. All traversed shards are returned, starting with the passed
|
|
339
|
-
* shard and ending with the target.
|
|
340
|
-
*
|
|
341
|
-
* @param {ShardFetcher} shards
|
|
342
|
-
* @param {API.ShardBlockView} shard
|
|
343
|
-
* @param {string} key
|
|
344
|
-
* @returns {Promise<[API.ShardBlockView, ...Array<API.ShardBlockView>]>}
|
|
345
|
-
*/
|
|
346
|
-
const traverse = async (shards, shard, key) => {
|
|
347
|
-
for (const [k, v] of shard.value.entries) {
|
|
348
|
-
if (key === k)
|
|
349
|
-
return [shard];
|
|
350
|
-
if (key.startsWith(k) && Array.isArray(v)) {
|
|
351
|
-
const path = await traverse(shards, await shards.get(v[0]), key.slice(k.length));
|
|
352
|
-
return [shard, ...path];
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
return [shard];
|
|
356
|
-
};
|
package/dist/src/merge.d.ts
DELETED
package/dist/src/merge.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../src/merge.js"],"names":[],"mappings":"AAWO,8BALI,GAAG,CAAC,YAAY,QAChB,GAAG,CAAC,SAAS,WACb,GAAG,CAAC,SAAS,EAAE,GACb,OAAO,CAAC;IAAE,IAAI,EAAE,GAAG,CAAC,SAAS,CAAA;CAAE,GAAG,GAAG,CAAC,SAAS,CAAC,CAiC5D;qBAzCoB,UAAU"}
|
package/dist/src/merge.js
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
// eslint-disable-next-line no-unused-vars
|
|
2
|
-
import * as API from './api.js';
|
|
3
|
-
import { difference } from './diff.js';
|
|
4
|
-
import { put, del } from './index.js';
|
|
5
|
-
/**
|
|
6
|
-
* @param {API.BlockFetcher} blocks Bucket block storage.
|
|
7
|
-
* @param {API.ShardLink} base Merge base. Common parent of target DAGs.
|
|
8
|
-
* @param {API.ShardLink[]} targets Target DAGs to merge.
|
|
9
|
-
* @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>}
|
|
10
|
-
*/
|
|
11
|
-
export const merge = async (blocks, base, targets) => {
|
|
12
|
-
const diffs = await Promise.all(targets.map(t => difference(blocks, base, t)));
|
|
13
|
-
const additions = new Map();
|
|
14
|
-
const removals = new Map();
|
|
15
|
-
/** @type {API.BlockFetcher} */
|
|
16
|
-
const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) };
|
|
17
|
-
let root = base;
|
|
18
|
-
for (const { keys } of diffs) {
|
|
19
|
-
for (const [k, v] of keys) {
|
|
20
|
-
let res;
|
|
21
|
-
if (v[1] == null) {
|
|
22
|
-
res = await del(fetcher, root, k);
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
res = await put(fetcher, root, k, v[1]);
|
|
26
|
-
}
|
|
27
|
-
for (const blk of res.removals) {
|
|
28
|
-
if (additions.has(blk.cid.toString())) {
|
|
29
|
-
additions.delete(blk.cid.toString());
|
|
30
|
-
}
|
|
31
|
-
else {
|
|
32
|
-
removals.set(blk.cid.toString(), blk);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
for (const blk of res.additions) {
|
|
36
|
-
additions.set(blk.cid.toString(), blk);
|
|
37
|
-
}
|
|
38
|
-
root = res.root;
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
return { root, additions: [...additions.values()], removals: [...removals.values()] };
|
|
42
|
-
};
|
package/dist/src/shard.d.ts
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
export const KeyCharsASCII: "ascii";
|
|
2
|
-
export const MaxKeySize: 4096;
|
|
3
|
-
/**
|
|
4
|
-
* @extends {Block<API.Shard, typeof dagCBOR.code, typeof sha256.code, 1>}
|
|
5
|
-
* @implements {API.ShardBlockView}
|
|
6
|
-
*/
|
|
7
|
-
export class ShardBlock extends Block<API.Shard, 113, 18, 1> implements API.ShardBlockView {
|
|
8
|
-
/** @param {API.ShardOptions} [options] */
|
|
9
|
-
static create(options?: API.ShardOptions): Promise<API.ShardBlockView>;
|
|
10
|
-
/**
|
|
11
|
-
* @param {object} config
|
|
12
|
-
* @param {API.ShardLink} config.cid
|
|
13
|
-
* @param {API.Shard} config.value
|
|
14
|
-
* @param {Uint8Array} config.bytes
|
|
15
|
-
*/
|
|
16
|
-
constructor({ cid, value, bytes }: {
|
|
17
|
-
cid: API.ShardLink;
|
|
18
|
-
value: API.Shard;
|
|
19
|
-
bytes: Uint8Array;
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
export function create(options?: API.ShardOptions): API.Shard;
|
|
23
|
-
export function configure(options?: API.ShardOptions): API.ShardConfig;
|
|
24
|
-
export function withEntries(entries: API.ShardEntry[], options?: API.ShardOptions): API.Shard;
|
|
25
|
-
export function encodeBlock(value: API.Shard): Promise<API.ShardBlockView>;
|
|
26
|
-
export function decodeBlock(bytes: Uint8Array): Promise<API.ShardBlockView>;
|
|
27
|
-
export function isShard(value: any): value is API.Shard;
|
|
28
|
-
export function isShardLink(value: any): value is API.ShardLink;
|
|
29
|
-
export class ShardFetcher {
|
|
30
|
-
/** @param {API.BlockFetcher} blocks */
|
|
31
|
-
constructor(blocks: API.BlockFetcher);
|
|
32
|
-
_blocks: API.BlockFetcher;
|
|
33
|
-
/**
|
|
34
|
-
* @param {API.ShardLink} link
|
|
35
|
-
* @returns {Promise<API.ShardBlockView>}
|
|
36
|
-
*/
|
|
37
|
-
get(link: API.ShardLink): Promise<API.ShardBlockView>;
|
|
38
|
-
}
|
|
39
|
-
export function putEntry(target: API.ShardEntry[], newEntry: API.ShardEntry): API.ShardEntry[];
|
|
40
|
-
export function isPrintableASCII(s: string): boolean;
|
|
41
|
-
import * as API from './api.js';
|
|
42
|
-
import { Block } from 'multiformats/block';
|
|
43
|
-
//# sourceMappingURL=shard.d.ts.map
|
package/dist/src/shard.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shard.d.ts","sourceRoot":"","sources":["../../src/shard.js"],"names":[],"mappings":"AAOA,4BAA6B,OAAO,CAAA;AACpC,yBAA0B,IAAI,CAAA;AAE9B;;;GAGG;AACH,wEAFgB,GAAG,CAAC,cAAc;IAchC,0CAA0C;IAC1C,wBADY,GAAG,CAAC,YAAY,+BAG3B;IAdD;;;;;OAKG;IACH,mCAJG;QAA8B,GAAG,EAAzB,GAAG,CAAC,SAAS;QACK,KAAK,EAAvB,GAAG,CAAC,KAAK;QACU,KAAK,EAAxB,UAAU;KACpB,EAIA;CAMF;AAMM,iCAHI,GAAG,CAAC,YAAY,GACd,GAAG,CAAC,KAAK,CAEqD;AAMpE,oCAHI,GAAG,CAAC,YAAY,GACd,GAAG,CAAC,WAAW,CAO1B;AAOK,qCAJI,GAAG,CAAC,UAAU,EAAE,YAChB,GAAG,CAAC,YAAY,GACd,GAAG,CAAC,KAAK,CAE4D;AAS3E,mCAHI,GAAG,CAAC,KAAK,GACP,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAOvC;AAMM,mCAHI,UAAU,GACR,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAQvC;AAMM,+BAHI,GAAG,GACD,KAAK,IAAI,GAAG,CAAC,KAAK,CASG;AAM3B,mCAHI,GAAG,GACD,KAAK,IAAI,GAAG,CAAC,SAAS,CAIN;AAE7B;IACE,uCAAuC;IACvC,oBADY,GAAG,CAAC,YAAY,EAG3B;IADC,0BAAqB;IAGvB;;;OAGG;IACH,UAHW,GAAG,CAAC,SAAS,GACX,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAMvC;CACF;AAOM,iCAJI,GAAG,CAAC,UAAU,EAAE,YAChB,GAAG,CAAC,UAAU,GACZ,GAAG,CAAC,UAAU,EAAE,CAqD5B;AAGM,oCADK,MAAM,WAC2C;qBA9KxC,UAAU;sBAJO,oBAAoB"}
|