cojson 0.8.0 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +11 -3
- package/CHANGELOG.md +13 -0
- package/dist/native/PeerState.js.map +1 -0
- package/dist/native/PriorityBasedMessageQueue.js.map +1 -0
- package/dist/native/base64url.js.map +1 -0
- package/dist/native/base64url.test.js.map +1 -0
- package/dist/native/coValue.js.map +1 -0
- package/dist/native/coValueCore.js.map +1 -0
- package/dist/native/coValues/account.js.map +1 -0
- package/dist/native/coValues/coList.js.map +1 -0
- package/dist/native/coValues/coMap.js.map +1 -0
- package/dist/native/coValues/coStream.js.map +1 -0
- package/dist/native/coValues/group.js.map +1 -0
- package/dist/native/coreToCoValue.js.map +1 -0
- package/dist/{crypto → native/crypto}/PureJSCrypto.js +4 -0
- package/dist/native/crypto/PureJSCrypto.js.map +1 -0
- package/dist/{crypto → native/crypto}/crypto.js +1 -1
- package/dist/native/crypto/crypto.js.map +1 -0
- package/dist/native/ids.js.map +1 -0
- package/dist/{index.js → native/index.native.js} +3 -4
- package/dist/native/index.native.js.map +1 -0
- package/dist/native/jsonStringify.js.map +1 -0
- package/dist/{jsonValue.js.map → native/jsonValue.js.map} +1 -1
- package/dist/{localNode.js → native/localNode.js} +50 -33
- package/dist/native/localNode.js.map +1 -0
- package/dist/native/media.js.map +1 -0
- package/dist/native/permissions.js.map +1 -0
- package/dist/native/priority.js.map +1 -0
- package/dist/native/storage/FileSystem.js.map +1 -0
- package/dist/{storage → native/storage}/chunksAndKnownStates.js +1 -1
- package/dist/native/storage/chunksAndKnownStates.js.map +1 -0
- package/dist/native/storage/index.js.map +1 -0
- package/dist/native/streamUtils.js.map +1 -0
- package/dist/native/sync.js.map +1 -0
- package/dist/native/typeUtils/accountOrAgentIDfromSessionID.js.map +1 -0
- package/dist/native/typeUtils/expectGroup.js.map +1 -0
- package/dist/native/typeUtils/isAccountID.js.map +1 -0
- package/dist/native/typeUtils/isCoValue.js.map +1 -0
- package/dist/web/PeerState.js +66 -0
- package/dist/web/PeerState.js.map +1 -0
- package/dist/web/PriorityBasedMessageQueue.js +51 -0
- package/dist/web/PriorityBasedMessageQueue.js.map +1 -0
- package/dist/web/base64url.js +59 -0
- package/dist/web/base64url.js.map +1 -0
- package/dist/web/base64url.test.js +25 -0
- package/dist/web/base64url.test.js.map +1 -0
- package/dist/web/coValue.js +19 -0
- package/dist/web/coValue.js.map +1 -0
- package/dist/web/coValueCore.js +693 -0
- package/dist/web/coValueCore.js.map +1 -0
- package/dist/web/coValues/account.js +96 -0
- package/dist/web/coValues/account.js.map +1 -0
- package/dist/web/coValues/coList.js +393 -0
- package/dist/web/coValues/coList.js.map +1 -0
- package/dist/web/coValues/coMap.js +197 -0
- package/dist/web/coValues/coMap.js.map +1 -0
- package/dist/web/coValues/coStream.js +220 -0
- package/dist/web/coValues/coStream.js.map +1 -0
- package/dist/web/coValues/group.js +250 -0
- package/dist/web/coValues/group.js.map +1 -0
- package/dist/web/coreToCoValue.js +42 -0
- package/dist/web/coreToCoValue.js.map +1 -0
- package/dist/web/crypto/PureJSCrypto.js +93 -0
- package/dist/web/crypto/PureJSCrypto.js.map +1 -0
- package/dist/{crypto → web/crypto}/WasmCrypto.js +3 -0
- package/dist/web/crypto/WasmCrypto.js.map +1 -0
- package/dist/web/crypto/crypto.js +154 -0
- package/dist/web/crypto/crypto.js.map +1 -0
- package/dist/web/ids.js +17 -0
- package/dist/web/ids.js.map +1 -0
- package/dist/web/index.web.js +40 -0
- package/dist/web/index.web.js.map +1 -0
- package/dist/web/jsonStringify.js +57 -0
- package/dist/web/jsonStringify.js.map +1 -0
- package/dist/web/jsonValue.js +2 -0
- package/dist/web/jsonValue.js.map +1 -0
- package/dist/web/localNode.js +453 -0
- package/dist/web/localNode.js.map +1 -0
- package/dist/web/media.js +2 -0
- package/dist/web/media.js.map +1 -0
- package/dist/web/permissions.js +206 -0
- package/dist/web/permissions.js.map +1 -0
- package/dist/web/priority.js +31 -0
- package/dist/web/priority.js.map +1 -0
- package/dist/web/storage/FileSystem.js +58 -0
- package/dist/web/storage/FileSystem.js.map +1 -0
- package/dist/web/storage/chunksAndKnownStates.js +100 -0
- package/dist/web/storage/chunksAndKnownStates.js.map +1 -0
- package/dist/web/storage/index.js +348 -0
- package/dist/web/storage/index.js.map +1 -0
- package/dist/web/streamUtils.js +41 -0
- package/dist/web/streamUtils.js.map +1 -0
- package/dist/web/sync.js +504 -0
- package/dist/web/sync.js.map +1 -0
- package/dist/web/typeUtils/accountOrAgentIDfromSessionID.js +5 -0
- package/dist/web/typeUtils/accountOrAgentIDfromSessionID.js.map +1 -0
- package/dist/web/typeUtils/expectGroup.js +13 -0
- package/dist/web/typeUtils/expectGroup.js.map +1 -0
- package/dist/web/typeUtils/isAccountID.js +4 -0
- package/dist/web/typeUtils/isAccountID.js.map +1 -0
- package/dist/web/typeUtils/isCoValue.js +11 -0
- package/dist/web/typeUtils/isCoValue.js.map +1 -0
- package/package.json +25 -8
- package/src/coValue.ts +1 -1
- package/src/coValueCore.ts +4 -1
- package/src/coValues/account.ts +9 -3
- package/src/crypto/PureJSCrypto.ts +5 -0
- package/src/crypto/WasmCrypto.ts +4 -0
- package/src/crypto/crypto.ts +7 -2
- package/src/{index.ts → index.native.ts} +4 -5
- package/src/index.web.ts +152 -0
- package/src/jsonValue.ts +25 -0
- package/src/localNode.ts +74 -55
- package/src/storage/chunksAndKnownStates.ts +1 -1
- package/src/storage/index.ts +1 -1
- package/src/tests/coList.test.ts +1 -1
- package/src/tests/coMap.test.ts +1 -1
- package/src/tests/coStream.test.ts +2 -1
- package/src/tests/cryptoImpl.test.ts +24 -2
- package/src/tests/group.test.ts +6 -8
- package/src/tests/permissions.test.ts +66 -18
- package/src/tests/priority.test.ts +44 -15
- package/tsconfig.json +5 -4
- package/tsconfig.native.json +12 -0
- package/tsconfig.web.json +11 -0
- package/dist/PeerState.js.map +0 -1
- package/dist/PriorityBasedMessageQueue.js.map +0 -1
- package/dist/base64url.js.map +0 -1
- package/dist/base64url.test.js.map +0 -1
- package/dist/coValue.js.map +0 -1
- package/dist/coValueCore.js.map +0 -1
- package/dist/coValues/account.js.map +0 -1
- package/dist/coValues/coList.js.map +0 -1
- package/dist/coValues/coMap.js.map +0 -1
- package/dist/coValues/coStream.js.map +0 -1
- package/dist/coValues/group.js.map +0 -1
- package/dist/coreToCoValue.js.map +0 -1
- package/dist/crypto/PureJSCrypto.js.map +0 -1
- package/dist/crypto/WasmCrypto.js.map +0 -1
- package/dist/crypto/crypto.js.map +0 -1
- package/dist/ids.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/jsonStringify.js.map +0 -1
- package/dist/localNode.js.map +0 -1
- package/dist/media.js.map +0 -1
- package/dist/permissions.js.map +0 -1
- package/dist/priority.js.map +0 -1
- package/dist/storage/FileSystem.js.map +0 -1
- package/dist/storage/chunksAndKnownStates.js.map +0 -1
- package/dist/storage/index.js.map +0 -1
- package/dist/streamUtils.js.map +0 -1
- package/dist/sync.js.map +0 -1
- package/dist/tests/PeerState.test.js +0 -80
- package/dist/tests/PeerState.test.js.map +0 -1
- package/dist/tests/PriorityBasedMessageQueue.test.js +0 -97
- package/dist/tests/PriorityBasedMessageQueue.test.js.map +0 -1
- package/dist/tests/account.test.js +0 -58
- package/dist/tests/account.test.js.map +0 -1
- package/dist/tests/coList.test.js +0 -76
- package/dist/tests/coList.test.js.map +0 -1
- package/dist/tests/coMap.test.js +0 -136
- package/dist/tests/coMap.test.js.map +0 -1
- package/dist/tests/coStream.test.js +0 -205
- package/dist/tests/coStream.test.js.map +0 -1
- package/dist/tests/coValueCore.test.js +0 -124
- package/dist/tests/coValueCore.test.js.map +0 -1
- package/dist/tests/crypto.test.js +0 -118
- package/dist/tests/crypto.test.js.map +0 -1
- package/dist/tests/cryptoImpl.test.js +0 -113
- package/dist/tests/cryptoImpl.test.js.map +0 -1
- package/dist/tests/group.test.js +0 -34
- package/dist/tests/group.test.js.map +0 -1
- package/dist/tests/permissions.test.js +0 -1059
- package/dist/tests/permissions.test.js.map +0 -1
- package/dist/tests/priority.test.js +0 -61
- package/dist/tests/priority.test.js.map +0 -1
- package/dist/tests/sync.test.js +0 -1186
- package/dist/tests/sync.test.js.map +0 -1
- package/dist/tests/testUtils.js +0 -59
- package/dist/tests/testUtils.js.map +0 -1
- package/dist/typeUtils/accountOrAgentIDfromSessionID.js.map +0 -1
- package/dist/typeUtils/expectGroup.js.map +0 -1
- package/dist/typeUtils/isAccountID.js.map +0 -1
- package/dist/typeUtils/isCoValue.js.map +0 -1
- /package/dist/{PeerState.js → native/PeerState.js} +0 -0
- /package/dist/{PriorityBasedMessageQueue.js → native/PriorityBasedMessageQueue.js} +0 -0
- /package/dist/{base64url.js → native/base64url.js} +0 -0
- /package/dist/{base64url.test.js → native/base64url.test.js} +0 -0
- /package/dist/{coValue.js → native/coValue.js} +0 -0
- /package/dist/{coValueCore.js → native/coValueCore.js} +0 -0
- /package/dist/{coValues → native/coValues}/account.js +0 -0
- /package/dist/{coValues → native/coValues}/coList.js +0 -0
- /package/dist/{coValues → native/coValues}/coMap.js +0 -0
- /package/dist/{coValues → native/coValues}/coStream.js +0 -0
- /package/dist/{coValues → native/coValues}/group.js +0 -0
- /package/dist/{coreToCoValue.js → native/coreToCoValue.js} +0 -0
- /package/dist/{ids.js → native/ids.js} +0 -0
- /package/dist/{jsonStringify.js → native/jsonStringify.js} +0 -0
- /package/dist/{jsonValue.js → native/jsonValue.js} +0 -0
- /package/dist/{media.js → native/media.js} +0 -0
- /package/dist/{permissions.js → native/permissions.js} +0 -0
- /package/dist/{priority.js → native/priority.js} +0 -0
- /package/dist/{storage → native/storage}/FileSystem.js +0 -0
- /package/dist/{storage → native/storage}/index.js +0 -0
- /package/dist/{streamUtils.js → native/streamUtils.js} +0 -0
- /package/dist/{sync.js → native/sync.js} +0 -0
- /package/dist/{typeUtils → native/typeUtils}/accountOrAgentIDfromSessionID.js +0 -0
- /package/dist/{typeUtils → native/typeUtils}/expectGroup.js +0 -0
- /package/dist/{typeUtils → native/typeUtils}/isAccountID.js +0 -0
- /package/dist/{typeUtils → native/typeUtils}/isCoValue.js +0 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
import { connectedPeers } from "../streamUtils.js";
|
|
2
|
+
import { chunkToKnownState, contentSinceChunk, mergeChunks, } from "./chunksAndKnownStates.js";
|
|
3
|
+
import { readChunk, readHeader, textDecoder, writeBlock, writeToWal, } from "./FileSystem.js";
|
|
4
|
+
const MAX_N_LEVELS = 3;
|
|
5
|
+
export class LSMStorage {
|
|
6
|
+
constructor(fs, fromLocalNode, toLocalNode) {
|
|
7
|
+
this.fs = fs;
|
|
8
|
+
this.fromLocalNode = fromLocalNode;
|
|
9
|
+
this.toLocalNode = toLocalNode;
|
|
10
|
+
this.headerCache = new Map();
|
|
11
|
+
this.blockFileHandles = new Map();
|
|
12
|
+
this.coValues = {};
|
|
13
|
+
this.currentWal = undefined;
|
|
14
|
+
let nMsg = 0;
|
|
15
|
+
const processMessages = async () => {
|
|
16
|
+
for await (const msg of fromLocalNode) {
|
|
17
|
+
console.log("Storage msg start", nMsg);
|
|
18
|
+
try {
|
|
19
|
+
if (msg === "Disconnected" || msg === "PingTimeout") {
|
|
20
|
+
throw new Error("Unexpected Disconnected message");
|
|
21
|
+
}
|
|
22
|
+
if (msg.action === "done") {
|
|
23
|
+
return;
|
|
24
|
+
}
|
|
25
|
+
if (msg.action === "content") {
|
|
26
|
+
await this.handleNewContent(msg);
|
|
27
|
+
}
|
|
28
|
+
else {
|
|
29
|
+
await this.sendNewContent(msg.id, msg, undefined);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
catch (e) {
|
|
33
|
+
console.error(new Error(`Error reading from localNode, handling msg\n\n${JSON.stringify(msg, (k, v) => k === "changes" || k === "encryptedChanges"
|
|
34
|
+
? v.slice(0, 20) + "..."
|
|
35
|
+
: v)}`, { cause: e }));
|
|
36
|
+
}
|
|
37
|
+
console.log("Storage msg end", nMsg);
|
|
38
|
+
nMsg++;
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
processMessages().catch((e) => console.error("Error in processMessages in storage", e));
|
|
42
|
+
setTimeout(() => this.compact().catch((e) => {
|
|
43
|
+
console.error("Error while compacting", e);
|
|
44
|
+
}), 20000);
|
|
45
|
+
}
|
|
46
|
+
async sendNewContent(id, known, asDependencyOf) {
|
|
47
|
+
let coValue = this.coValues[id];
|
|
48
|
+
if (!coValue) {
|
|
49
|
+
coValue = await this.loadCoValue(id, this.fs);
|
|
50
|
+
}
|
|
51
|
+
if (!coValue) {
|
|
52
|
+
this.toLocalNode
|
|
53
|
+
.push({
|
|
54
|
+
id: id,
|
|
55
|
+
action: "known",
|
|
56
|
+
header: false,
|
|
57
|
+
sessions: {},
|
|
58
|
+
asDependencyOf,
|
|
59
|
+
})
|
|
60
|
+
.catch((e) => console.error("Error while pushing known", e));
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
if (!known?.header && coValue.header?.ruleset.type === "ownedByGroup") {
|
|
64
|
+
await this.sendNewContent(coValue.header.ruleset.group, undefined, asDependencyOf || id);
|
|
65
|
+
}
|
|
66
|
+
else if (!known?.header && coValue.header?.ruleset.type === "group") {
|
|
67
|
+
const dependedOnAccounts = new Set();
|
|
68
|
+
for (const session of Object.values(coValue.sessionEntries)) {
|
|
69
|
+
for (const entry of session) {
|
|
70
|
+
for (const tx of entry.transactions) {
|
|
71
|
+
if (tx.privacy === "trusting") {
|
|
72
|
+
const parsedChanges = JSON.parse(tx.changes);
|
|
73
|
+
for (const change of parsedChanges) {
|
|
74
|
+
if (change.op === "set" &&
|
|
75
|
+
change.key.startsWith("co_")) {
|
|
76
|
+
dependedOnAccounts.add(change.key);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
for (const account of dependedOnAccounts) {
|
|
84
|
+
await this.sendNewContent(account, undefined, asDependencyOf || id);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
const newContentMessages = contentSinceChunk(id, coValue, known).map((message) => ({ ...message, asDependencyOf }));
|
|
88
|
+
const ourKnown = chunkToKnownState(id, coValue);
|
|
89
|
+
this.toLocalNode
|
|
90
|
+
.push({
|
|
91
|
+
action: "known",
|
|
92
|
+
...ourKnown,
|
|
93
|
+
asDependencyOf,
|
|
94
|
+
})
|
|
95
|
+
.catch((e) => console.error("Error while pushing known", e));
|
|
96
|
+
for (const message of newContentMessages) {
|
|
97
|
+
if (Object.keys(message.new).length === 0)
|
|
98
|
+
continue;
|
|
99
|
+
this.toLocalNode
|
|
100
|
+
.push(message)
|
|
101
|
+
.catch((e) => console.error("Error while pushing new content", e));
|
|
102
|
+
}
|
|
103
|
+
this.coValues[id] = coValue;
|
|
104
|
+
}
|
|
105
|
+
async withWAL(handler) {
|
|
106
|
+
if (!this.currentWal) {
|
|
107
|
+
this.currentWal = await this.fs.createFile(`wal-${Date.now()}-${Math.random()
|
|
108
|
+
.toString(36)
|
|
109
|
+
.slice(2)}.jsonl`);
|
|
110
|
+
}
|
|
111
|
+
await handler(this.currentWal);
|
|
112
|
+
}
|
|
113
|
+
async handleNewContent(newContent) {
|
|
114
|
+
const coValue = this.coValues[newContent.id];
|
|
115
|
+
const newContentAsChunk = {
|
|
116
|
+
header: newContent.header,
|
|
117
|
+
sessionEntries: Object.fromEntries(Object.entries(newContent.new).map(([sessionID, newInSession]) => [
|
|
118
|
+
sessionID,
|
|
119
|
+
[
|
|
120
|
+
{
|
|
121
|
+
after: newInSession.after,
|
|
122
|
+
lastSignature: newInSession.lastSignature,
|
|
123
|
+
transactions: newInSession.newTransactions,
|
|
124
|
+
},
|
|
125
|
+
],
|
|
126
|
+
])),
|
|
127
|
+
};
|
|
128
|
+
if (!coValue) {
|
|
129
|
+
if (newContent.header) {
|
|
130
|
+
// console.log("Creating in WAL", newContent.id);
|
|
131
|
+
await this.withWAL((wal) => writeToWal(wal, this.fs, newContent.id, newContentAsChunk));
|
|
132
|
+
this.coValues[newContent.id] = newContentAsChunk;
|
|
133
|
+
}
|
|
134
|
+
else {
|
|
135
|
+
console.warn("Incontiguous incoming update for " + newContent.id);
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
else {
|
|
140
|
+
const merged = mergeChunks(coValue, newContentAsChunk);
|
|
141
|
+
if (merged === "nonContigous") {
|
|
142
|
+
console.warn("Non-contigous new content for " + newContent.id, Object.entries(coValue.sessionEntries).map(([session, entries]) => entries.map((entry) => ({
|
|
143
|
+
session: session,
|
|
144
|
+
after: entry.after,
|
|
145
|
+
length: entry.transactions.length,
|
|
146
|
+
}))), Object.entries(newContentAsChunk.sessionEntries).map(([session, entries]) => entries.map((entry) => ({
|
|
147
|
+
session: session,
|
|
148
|
+
after: entry.after,
|
|
149
|
+
length: entry.transactions.length,
|
|
150
|
+
}))));
|
|
151
|
+
}
|
|
152
|
+
else {
|
|
153
|
+
// console.log("Appending to WAL", newContent.id);
|
|
154
|
+
await this.withWAL((wal) => writeToWal(wal, this.fs, newContent.id, newContentAsChunk));
|
|
155
|
+
this.coValues[newContent.id] = merged;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
async getBlockHandle(blockFile, fs) {
|
|
160
|
+
if (!this.blockFileHandles.has(blockFile)) {
|
|
161
|
+
this.blockFileHandles.set(blockFile, fs.openToRead(blockFile));
|
|
162
|
+
}
|
|
163
|
+
return this.blockFileHandles.get(blockFile);
|
|
164
|
+
}
|
|
165
|
+
async loadCoValue(id, fs) {
|
|
166
|
+
const files = this.fileCache || (await fs.listFiles());
|
|
167
|
+
this.fileCache = files;
|
|
168
|
+
const blockFiles = files.filter((name) => name.startsWith("L")).sort();
|
|
169
|
+
let result;
|
|
170
|
+
for (const blockFile of blockFiles) {
|
|
171
|
+
let cachedHeader = this.headerCache.get(blockFile);
|
|
172
|
+
const { handle, size } = await this.getBlockHandle(blockFile, fs);
|
|
173
|
+
// console.log("Attempting to load", id, blockFile);
|
|
174
|
+
if (!cachedHeader) {
|
|
175
|
+
cachedHeader = {};
|
|
176
|
+
const header = await readHeader(blockFile, handle, size, fs);
|
|
177
|
+
for (const entry of header) {
|
|
178
|
+
cachedHeader[entry.id] = {
|
|
179
|
+
start: entry.start,
|
|
180
|
+
length: entry.length,
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
this.headerCache.set(blockFile, cachedHeader);
|
|
184
|
+
}
|
|
185
|
+
const headerEntry = cachedHeader[id];
|
|
186
|
+
// console.log("Header entry", id, headerEntry);
|
|
187
|
+
if (headerEntry) {
|
|
188
|
+
const nextChunk = await readChunk(handle, headerEntry, fs);
|
|
189
|
+
if (result) {
|
|
190
|
+
const merged = mergeChunks(result, nextChunk);
|
|
191
|
+
if (merged === "nonContigous") {
|
|
192
|
+
console.warn("Non-contigous chunks while loading " + id, result, nextChunk);
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
result = merged;
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
result = nextChunk;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
// await fs.close(handle);
|
|
203
|
+
}
|
|
204
|
+
return result;
|
|
205
|
+
}
|
|
206
|
+
async compact() {
|
|
207
|
+
const fileNames = await this.fs.listFiles();
|
|
208
|
+
const walFiles = fileNames.filter((name) => name.startsWith("wal-"));
|
|
209
|
+
walFiles.sort();
|
|
210
|
+
const coValues = new Map();
|
|
211
|
+
console.log("Compacting WAL files", walFiles);
|
|
212
|
+
if (walFiles.length === 0)
|
|
213
|
+
return;
|
|
214
|
+
const oldWal = this.currentWal;
|
|
215
|
+
this.currentWal = undefined;
|
|
216
|
+
if (oldWal) {
|
|
217
|
+
await this.fs.close(oldWal);
|
|
218
|
+
}
|
|
219
|
+
for (const fileName of walFiles) {
|
|
220
|
+
const { handle, size } = await this.fs.openToRead(fileName);
|
|
221
|
+
if (size === 0) {
|
|
222
|
+
await this.fs.close(handle);
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
const bytes = await this.fs.read(handle, 0, size);
|
|
226
|
+
const decoded = textDecoder.decode(bytes);
|
|
227
|
+
const lines = decoded.split("\n");
|
|
228
|
+
for (const line of lines) {
|
|
229
|
+
if (line.length === 0)
|
|
230
|
+
continue;
|
|
231
|
+
const chunk = JSON.parse(line);
|
|
232
|
+
const existingChunk = coValues.get(chunk.id);
|
|
233
|
+
if (existingChunk) {
|
|
234
|
+
const merged = mergeChunks(existingChunk, chunk);
|
|
235
|
+
if (merged === "nonContigous") {
|
|
236
|
+
console.log("Non-contigous chunks in " +
|
|
237
|
+
chunk.id +
|
|
238
|
+
", " +
|
|
239
|
+
fileName, existingChunk, chunk);
|
|
240
|
+
}
|
|
241
|
+
else {
|
|
242
|
+
coValues.set(chunk.id, merged);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
else {
|
|
246
|
+
coValues.set(chunk.id, chunk);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
await this.fs.close(handle);
|
|
250
|
+
}
|
|
251
|
+
const highestBlockNumber = fileNames.reduce((acc, name) => {
|
|
252
|
+
if (name.startsWith("L" + MAX_N_LEVELS)) {
|
|
253
|
+
const num = parseInt(name.split("-")[1]);
|
|
254
|
+
if (num > acc) {
|
|
255
|
+
return num;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
return acc;
|
|
259
|
+
}, 0);
|
|
260
|
+
console.log([...coValues.keys()], fileNames, highestBlockNumber);
|
|
261
|
+
await writeBlock(coValues, MAX_N_LEVELS, highestBlockNumber + 1, this.fs);
|
|
262
|
+
for (const walFile of walFiles) {
|
|
263
|
+
await this.fs.removeFile(walFile);
|
|
264
|
+
}
|
|
265
|
+
this.fileCache = undefined;
|
|
266
|
+
const fileNames2 = await this.fs.listFiles();
|
|
267
|
+
const blockFiles = fileNames2.filter((name) => name.startsWith("L")).sort();
|
|
268
|
+
const blockFilesByLevelInOrder = {};
|
|
269
|
+
for (const blockFile of blockFiles) {
|
|
270
|
+
const level = parseInt(blockFile.split("-")[0].slice(1));
|
|
271
|
+
if (!blockFilesByLevelInOrder[level]) {
|
|
272
|
+
blockFilesByLevelInOrder[level] = [];
|
|
273
|
+
}
|
|
274
|
+
blockFilesByLevelInOrder[level].push(blockFile);
|
|
275
|
+
}
|
|
276
|
+
console.log(blockFilesByLevelInOrder);
|
|
277
|
+
for (let level = MAX_N_LEVELS; level > 0; level--) {
|
|
278
|
+
const nBlocksDesired = Math.pow(2, level);
|
|
279
|
+
const blocksInLevel = blockFilesByLevelInOrder[level];
|
|
280
|
+
if (blocksInLevel && blocksInLevel.length > nBlocksDesired) {
|
|
281
|
+
console.log("Compacting blocks in level", level, blocksInLevel);
|
|
282
|
+
const coValues = new Map();
|
|
283
|
+
for (const blockFile of blocksInLevel) {
|
|
284
|
+
const { handle, size } = await this.getBlockHandle(blockFile, this.fs);
|
|
285
|
+
if (size === 0) {
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
const header = await readHeader(blockFile, handle, size, this.fs);
|
|
289
|
+
for (const entry of header) {
|
|
290
|
+
const chunk = await readChunk(handle, entry, this.fs);
|
|
291
|
+
const existingChunk = coValues.get(entry.id);
|
|
292
|
+
if (existingChunk) {
|
|
293
|
+
const merged = mergeChunks(existingChunk, chunk);
|
|
294
|
+
if (merged === "nonContigous") {
|
|
295
|
+
console.log("Non-contigous chunks in " +
|
|
296
|
+
entry.id +
|
|
297
|
+
", " +
|
|
298
|
+
blockFile, existingChunk, chunk);
|
|
299
|
+
}
|
|
300
|
+
else {
|
|
301
|
+
coValues.set(entry.id, merged);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
coValues.set(entry.id, chunk);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
let levelBelow = blockFilesByLevelInOrder[level - 1];
|
|
310
|
+
if (!levelBelow) {
|
|
311
|
+
levelBelow = [];
|
|
312
|
+
blockFilesByLevelInOrder[level - 1] = levelBelow;
|
|
313
|
+
}
|
|
314
|
+
const highestBlockNumberInLevelBelow = levelBelow.reduce((acc, name) => {
|
|
315
|
+
const num = parseInt(name.split("-")[1]);
|
|
316
|
+
if (num > acc) {
|
|
317
|
+
return num;
|
|
318
|
+
}
|
|
319
|
+
return acc;
|
|
320
|
+
}, 0);
|
|
321
|
+
const newBlockName = await writeBlock(coValues, level - 1, highestBlockNumberInLevelBelow + 1, this.fs);
|
|
322
|
+
levelBelow.push(newBlockName);
|
|
323
|
+
// delete blocks that went into this one
|
|
324
|
+
for (const blockFile of blocksInLevel) {
|
|
325
|
+
const handle = await this.getBlockHandle(blockFile, this.fs);
|
|
326
|
+
await this.fs.close(handle.handle);
|
|
327
|
+
await this.fs.removeFile(blockFile);
|
|
328
|
+
this.blockFileHandles.delete(blockFile);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
setTimeout(() => this.compact().catch((e) => {
|
|
333
|
+
console.error("Error while compacting", e);
|
|
334
|
+
}), 5000);
|
|
335
|
+
}
|
|
336
|
+
static asPeer({ fs, trace, localNodeName = "local", }) {
|
|
337
|
+
const [localNodeAsPeer, storageAsPeer] = connectedPeers(localNodeName, "storage", {
|
|
338
|
+
peer1role: "client",
|
|
339
|
+
peer2role: "server",
|
|
340
|
+
trace,
|
|
341
|
+
crashOnClose: true,
|
|
342
|
+
});
|
|
343
|
+
new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
|
|
344
|
+
// return { ...storageAsPeer, priority: 200 };
|
|
345
|
+
return storageAsPeer;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/storage/index.ts"],"names":[],"mappings":"AAWA,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,OAAO,EACH,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,GACd,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAKH,SAAS,EACT,UAAU,EACV,WAAW,EACX,UAAU,EACV,UAAU,GACb,MAAM,iBAAiB,CAAC;AAGzB,MAAM,YAAY,GAAG,CAAC,CAAC;AAavB,MAAM,OAAO,UAAU;IAenB,YACW,EAAM,EACN,aAAiC,EACjC,WAA8B;QAF9B,OAAE,GAAF,EAAE,CAAI;QACN,kBAAa,GAAb,aAAa,CAAoB;QACjC,gBAAW,GAAX,WAAW,CAAmB;QAZzC,gBAAW,GAAG,IAAI,GAAG,EAGlB,CAAC;QACJ,qBAAgB,GAAG,IAAI,GAAG,EAGvB,CAAC;QAOA,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YAC/B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;gBACvC,IAAI,CAAC;oBACD,IAAI,GAAG,KAAK,cAAc,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;wBAClD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACvD,CAAC;oBACD,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;wBACxB,OAAO;oBACX,CAAC;oBAED,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;wBAC3B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;oBACrC,CAAC;yBAAM,CAAC;wBACJ,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;oBACtD,CAAC;gBACL,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACT,OAAO,CAAC,KAAK,CACT,IAAI,KAAK,CACL,iDAAiD,IAAI,CAAC,SAAS,CAC3D,GAAG,EACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACL,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,kBAAkB;wBACvC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;wBACxB,CAAC,CAAC,CAAC,CACd,EAAE,EACH,EAAE,KAAK,EAAE,CAAC,EAAE,CACf,CACJ,CAAC;gBACN,CAAC;gBACD,OAAO,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;gBACrC,IAAI,EAAE,CAAC;YACX,CAAC;QACL,CAAC,CAAC;QAEF,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1B,OAAO,CAAC,KAAK,CAAC,qCAAqC,EAAE,CAAC,CAAC,CAC1D,CAAC;QAEF,UAAU,CACN,GAAG,EAAE,CACD,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,EACN,KAAK,CACR,CAAC;IACN,CAAC;IAED,KAAK,CAAC,cAAc,CAChB,EAAW,EACX,KAAoC,EACpC,cAAmC;QAEnC,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEhC,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAClD,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,CAAC,WAAW;iBACX,IAAI,CAAC;gBACF,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,EAAE;gBACZ,cAAc;aACjB,CAAC;iBACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,CAAC;YAEjE,OAAO;QACX,CAAC;QAED,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACpE,MAAM,IAAI,CAAC,cAAc,CACrB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAC5B,SAAS,EACT,cAAc,IAAI,EAAE,CACvB,CAAC;QACN,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACpE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAE,CAAC;YACrC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC1D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC1B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;wBAClC,IAAI,EAAE,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;4BAC5B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;4BAC7C,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gCACjC,IACI,MAAM,CAAC,EAAE,KAAK,KAAK;oCACnB,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAC9B,CAAC;oCACC,kBAAkB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gCACvC,CAAC;4BACL,CAAC;wBACL,CAAC;oBACL,CAAC;gBACL,CAAC;YACL,CAAC;YACD,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,cAAc,CACrB,OAA2B,EAC3B,SAAS,EACT,cAAc,IAAI,EAAE,CACvB,CAAC;YACN,CAAC;QACL,CAAC;QAED,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,GAAG,CAChE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,CAAC,CAChD,CAAC;QAEF,MAAM,QAAQ,GAAsB,iBAAiB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEnE,IAAI,CAAC,WAAW;aACX,IAAI,CAAC;YACF,MAAM,EAAE,OAAO;YACf,GAAG,QAAQ;YACX,cAAc;SACjB,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,2BAA2B,EAAE,CAAC,CAAC,CAAC,CAAC;QAEjE,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACvC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACpD,IAAI,CAAC,WAAW;iBACX,IAAI,CAAC,OAAO,CAAC;iBACb,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,OAAO,CAAC,KAAK,CAAC,iCAAiC,EAAE,CAAC,CAAC,CACtD,CAAC;QACV,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAmC;QAC7C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACnB,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CACtC,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE;iBAC7B,QAAQ,CAAC,EAAE,CAAC;iBACZ,KAAK,CAAC,CAAC,CAAC,QAAQ,CACxB,CAAC;QACN,CAAC;QACD,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAA6B;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAE7C,MAAM,iBAAiB,GAAiB;YACpC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,cAAc,EAAE,MAAM,CAAC,WAAW,CAC9B,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,CAC9B,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC;gBAC3B,SAAS;gBACT;oBACI;wBACI,KAAK,EAAE,YAAY,CAAC,KAAK;wBACzB,aAAa,EAAE,YAAY,CAAC,aAAa;wBACzC,YAAY,EAAE,YAAY,CAAC,eAAe;qBAC7C;iBACJ;aACJ,CACJ,CACJ;SACJ,CAAC;QAEF,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACpB,iDAAiD;gBACjD,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACvB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAC7D,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC;YACrD,CAAC;iBAAM,CAAC;gBACJ,OAAO,CAAC,IAAI,CACR,mCAAmC,GAAG,UAAU,CAAC,EAAE,CACtD,CAAC;gBACF,OAAO;YACX,CAAC;QACL,CAAC;aAAM,CAAC;YACJ,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACvD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CACR,gCAAgC,GAAG,UAAU,CAAC,EAAE,EAChD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CACtC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CACnB,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACpB,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;iBACpC,CAAC,CAAC,CACV,EACD,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,GAAG,CAChD,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CACnB,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACpB,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;iBACpC,CAAC,CAAC,CACV,CACJ,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,kDAAkD;gBAClD,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACvB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAC7D,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YAC1C,CAAC;QACL,CAAC;IACL,CAAC;IAED,KAAK,CAAC,cAAc,CAChB,SAAwB,EACxB,EAAM;QAEN,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;QACnE,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAW,EAAE,EAAM;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,UAAU,GACZ,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC9C,CAAC,IAAI,EAAE,CAAC;QAET,IAAI,MAAM,CAAC;QAEX,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACjC,IAAI,YAAY,GAEE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAElD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAElE,oDAAoD;YAEpD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAChB,YAAY,GAAG,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC7D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACzB,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG;wBACrB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACvB,CAAC;gBACN,CAAC;gBAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAClD,CAAC;YACD,MAAM,WAAW,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YAErC,gDAAgD;YAEhD,IAAI,WAAW,EAAE,CAAC;gBACd,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM,EAAE,CAAC;oBACT,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;oBAE9C,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC5B,OAAO,CAAC,IAAI,CACR,qCAAqC,GAAG,EAAE,EAC1C,MAAM,EACN,SAAS,CACZ,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,MAAM,GAAG,MAAM,CAAC;oBACpB,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,MAAM,GAAG,SAAS,CAAC;gBACvB,CAAC;YACL,CAAC;YAED,0BAA0B;QAC9B,CAAC;QAED,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,KAAK,CAAC,OAAO;QACT,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;QAE5C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACvC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CACT,CAAC;QACnB,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;QAElD,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,QAAQ,CAAC,CAAC;QAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,IAAI,MAAM,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAC9B,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAClB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACb,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC5B,SAAS;YACb,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YAElD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACvB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC;gBAE3C,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAE7C,IAAI,aAAa,EAAE,CAAC;oBAChB,MAAM,MAAM,GAAG,WAAW,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;oBACjD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC5B,OAAO,CAAC,GAAG,CACP,0BAA0B;4BACtB,KAAK,CAAC,EAAE;4BACR,IAAI;4BACJ,QAAQ,EACZ,aAAa,EACb,KAAK,CACR,CAAC;oBACN,CAAC;yBAAM,CAAC;wBACJ,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACnC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACJ,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAClC,CAAC;YACL,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,kBAAkB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACtD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,CAAC;gBACtC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;gBAC1C,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;oBACZ,OAAO,GAAG,CAAC;gBACf,CAAC;YACL,CAAC;YACD,OAAO,GAAG,CAAC;QACf,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,SAAS,EAAE,kBAAkB,CAAC,CAAC;QAEjE,MAAM,UAAU,CACZ,QAAQ,EACR,YAAY,EACZ,kBAAkB,GAAG,CAAC,EACtB,IAAI,CAAC,EAAE,CACV,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;QAE7C,MAAM,UAAU,GACZ,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CACnD,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,wBAAwB,GAE1B,EAAE,CAAC;QAEP,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACnC,wBAAwB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACzC,CAAC;YACD,wBAAwB,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAEtC,KAAK,IAAI,KAAK,GAAG,YAAY,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAChD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;YAEtD,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,4BAA4B,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;gBAEhE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;gBAElD,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;oBACpC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAClB,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAElD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACb,SAAS;oBACb,CAAC;oBACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAC3B,SAAS,EACT,MAAM,EACN,IAAI,EACJ,IAAI,CAAC,EAAE,CACV,CAAC;oBACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBACzB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;wBAEtD,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAE7C,IAAI,aAAa,EAAE,CAAC;4BAChB,MAAM,MAAM,GAAG,WAAW,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;4BACjD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gCAC5B,OAAO,CAAC,GAAG,CACP,0BAA0B;oCACtB,KAAK,CAAC,EAAE;oCACR,IAAI;oCACJ,SAAS,EACb,aAAa,EACb,KAAK,CACR,CAAC;4BACN,CAAC;iCAAM,CAAC;gCACJ,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BACnC,CAAC;wBACL,CAAC;6BAAM,CAAC;4BACJ,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;wBAClC,CAAC;oBACL,CAAC;gBACL,CAAC;gBAED,IAAI,UAAU,GAAG,wBAAwB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,UAAU,EAAE,CAAC;oBACd,UAAU,GAAG,EAAE,CAAC;oBAChB,wBAAwB,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;gBACrD,CAAC;gBAED,MAAM,8BAA8B,GAAG,UAAU,CAAC,MAAM,CACpD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;oBACV,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;oBAC1C,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;wBACZ,OAAO,GAAG,CAAC;oBACf,CAAC;oBACD,OAAO,GAAG,CAAC;gBACf,CAAC,EACD,CAAC,CACJ,CAAC;gBAEF,MAAM,YAAY,GAAG,MAAM,UAAU,CACjC,QAAQ,EACR,KAAK,GAAG,CAAC,EACT,8BAA8B,GAAG,CAAC,EAClC,IAAI,CAAC,EAAE,CACV,CAAC;gBACF,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAE9B,wCAAwC;gBACxC,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;oBACpC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CACpC,SAAS,EACT,IAAI,CAAC,EAAE,CACV,CAAC;oBACF,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBACpC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC5C,CAAC;YACL,CAAC;QACL,CAAC;QAED,UAAU,CACN,GAAG,EAAE,CACD,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACvB,OAAO,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,EACN,IAAI,CACP,CAAC;IACN,CAAC;IAED,MAAM,CAAC,MAAM,CAAwC,EACjD,EAAE,EACF,KAAK,EACL,aAAa,GAAG,OAAO,GAK1B;QACG,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,GAAG,cAAc,CACnD,aAAa,EACb,SAAS,EACT;YACI,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,QAAQ;YACnB,KAAK;YACL,YAAY,EAAE,IAAI;SACrB,CACJ,CAAC;QAEF,IAAI,UAAU,CAAC,EAAE,EAAE,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEvE,8CAA8C;QAC9C,OAAO,aAAa,CAAC;IACzB,CAAC;CACJ"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Channel } from "queueueue";
|
|
2
|
+
export { Channel } from "queueueue";
|
|
3
|
+
export function connectedPeers(peer1id, peer2id, { trace = false, peer1role = "peer", peer2role = "peer", crashOnClose = false, } = {}) {
|
|
4
|
+
const [from1to2Rx, from1to2Tx] = newQueuePair(trace ? { traceAs: `${peer1id} -> ${peer2id}` } : undefined);
|
|
5
|
+
const [from2to1Rx, from2to1Tx] = newQueuePair(trace ? { traceAs: `${peer2id} -> ${peer1id}` } : undefined);
|
|
6
|
+
const peer2AsPeer = {
|
|
7
|
+
id: peer2id,
|
|
8
|
+
incoming: from2to1Rx,
|
|
9
|
+
outgoing: from1to2Tx,
|
|
10
|
+
role: peer2role,
|
|
11
|
+
crashOnClose: crashOnClose,
|
|
12
|
+
};
|
|
13
|
+
const peer1AsPeer = {
|
|
14
|
+
id: peer1id,
|
|
15
|
+
incoming: from1to2Rx,
|
|
16
|
+
outgoing: from2to1Tx,
|
|
17
|
+
role: peer1role,
|
|
18
|
+
crashOnClose: crashOnClose,
|
|
19
|
+
};
|
|
20
|
+
return [peer1AsPeer, peer2AsPeer];
|
|
21
|
+
}
|
|
22
|
+
export function newQueuePair(options = {}) {
|
|
23
|
+
const channel = new Channel();
|
|
24
|
+
if (options.traceAs) {
|
|
25
|
+
return [
|
|
26
|
+
(async function* () {
|
|
27
|
+
for await (const msg of channel) {
|
|
28
|
+
console.debug(options.traceAs, JSON.stringify(msg, (k, v) => k === "changes" || k === "encryptedChanges"
|
|
29
|
+
? v.slice(0, 20) + "..."
|
|
30
|
+
: v, 2));
|
|
31
|
+
yield msg;
|
|
32
|
+
}
|
|
33
|
+
})(),
|
|
34
|
+
channel,
|
|
35
|
+
];
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
return [channel.wrap(), channel];
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=streamUtils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../../src/streamUtils.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,UAAU,cAAc,CAC1B,OAAe,EACf,OAAe,EACf,EACI,KAAK,GAAG,KAAK,EACb,SAAS,GAAG,MAAM,EAClB,SAAS,GAAG,MAAM,EAClB,YAAY,GAAG,KAAK,MAMpB,EAAE;IAEN,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG,YAAY,CACzC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,OAAO,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9D,CAAC;IACF,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG,YAAY,CACzC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,OAAO,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAC9D,CAAC;IAEF,MAAM,WAAW,GAAS;QACtB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,YAAY;KAC7B,CAAC;IAEF,MAAM,WAAW,GAAS;QACtB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,YAAY;KAC7B,CAAC;IAEF,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,YAAY,CACxB,UAAgC,EAAE;IAElC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAe,CAAC;IAE3C,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAClB,OAAO;YACH,CAAC,KAAK,SAAS,CAAC;gBACZ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;oBAC9B,OAAO,CAAC,KAAK,CACT,OAAO,CAAC,OAAO,EACf,IAAI,CAAC,SAAS,CACV,GAAG,EACH,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACL,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,kBAAkB;wBACvC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK;wBACxB,CAAC,CAAC,CAAC,EACX,CAAC,CACJ,CACJ,CAAC;oBACF,MAAM,GAAG,CAAC;gBACd,CAAC;YACL,CAAC,CAAC,EAAE;YACJ,OAAO;SACV,CAAC;IACN,CAAC;SAAM,CAAC;QACJ,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;IACrC,CAAC;AACL,CAAC"}
|