datafolder 1.0.5 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/data/sampleDataFolder/qykv/history/acl/1_acl.txt +1 -0
- package/data/sampleDataFolder/qykv/history/acl/2_acl.txt +1 -0
- package/data/sampleDataFolder/qykv/history/acl/3_acl.txt +1 -0
- package/data/sampleDataFolder/qykv/history/acl/4_acl.txt +1 -0
- package/data/sampleDataFolder/qykv/history/acl/5_acl.txt +1 -0
- package/data/sampleDataFolder/qykv/snapshot/1/1_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/1/1_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/1/1_vL.txt +10 -0
- package/data/sampleDataFolder/qykv/snapshot/2/1_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/2/1_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/2/1_vL.txt +1 -0
- package/data/sampleDataFolder/qykv/snapshot/3/1_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/3/1_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/3/1_vL.txt +1 -0
- package/data/sampleDataFolder/qykv/snapshot/4/1_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/4/1_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/4/1_vL.txt +1 -0
- package/data/sampleDataFolder/qykv/snapshot/4/2_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/4/2_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/4/2_vL.txt +9 -0
- package/data/sampleDataFolder/qykv/snapshot/5/1_pM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/5/1_prefixPM.txt +0 -0
- package/data/sampleDataFolder/qykv/snapshot/5/1_vL.txt +55 -0
- package/data/sampleDataFolder/qykv/snapshot/info/1.json +1 -0
- package/data/sampleDataFolder/qykv/snapshot/info/2.json +1 -0
- package/data/sampleDataFolder/qykv/snapshot/info/3.json +1 -0
- package/data/sampleDataFolder/qykv/snapshot/info/4.json +1 -0
- package/data/sampleDataFolder/qykv/snapshot/info/5.json +1 -0
- package/data/sampleDataFolder/qylog/2026/02/15/08/change.txt +58 -0
- package/data/sampleDataFolder/qylog/2026/02/15/08/stderr.txt +11 -0
- package/data/sampleDataFolder/qylog/2026/02/15/08/stdout.txt +36 -0
- package/package.json +1 -1
- package/sample.js +19 -19
- package/src/QyAcl.js +44 -44
- package/src/QyAclSaver_Worker.js +2 -2
- package/src/QyAclWriter.js +1 -1
- package/src/QyBatch.js +1 -3
- package/src/QyBinWriter.js +65 -66
- package/src/QyCD.js +63 -27
- package/src/QyCache.js +33 -33
- package/src/QyDB.js +4 -30
- package/src/QyDefaultOptions.js +2 -1
- package/src/QyFetch.js +4 -22
- package/src/QyFileLogger.js +2 -2
- package/src/QyKVData.js +159 -183
- package/src/QyKVDataCleaner_Worker.js +15 -14
- package/src/QyKVDataSaver_Worker.js +16 -16
- package/src/QyLogFileSaver.js +1 -1
- package/src/QyLogFileSaver_Worker.js +2 -2
- package/src/QyMod.js +7 -7
- package/src/QyOpRunner.js +19 -16
- package/src/QySnapshotCompactor_Worker.js +37 -36
- package/src/QySnapshotSaver_Worker.js +2 -2
- package/src/QySnapshots.js +141 -139
- package/src/QyUnionNode.js +2 -2
- package/src/rpc/evalCode.js +4 -4
- package/src/rpc/getFileContent.js +4 -4
- package/src/rpc/getSubItems.js +12 -12
- package/src/rpc/queryFiles.js +1 -1
- package/static/codemirror.bundle/editor.bundle.js +27906 -0
- package/static/datafolderui/QyUIFolderTree.js +121 -145
- package/static/datafolderui/QyUIUtils.js +9 -4
- package/static/datafolderui/code-mirror.js +35 -0
- package/static/datafolderui/display-tree.js +52 -86
- package/static/datafolderui/explore-tree.js +66 -81
- package/static/datafolderui/index.css +7 -0
- package/static/datafolderui/index.html +1 -0
- package/static/datafolderui/main-page.js +34 -55
- package/static/datafolderui/query-editor.js +178 -198
package/src/QyKVData.js
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "node:fs/promises";
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
-
getDefaultOptions as
|
|
10
|
+
getDefaultOptions as s
|
|
11
11
|
} from "./QyDefaultOptions.js";
|
|
12
12
|
|
|
13
13
|
import {
|
|
@@ -24,116 +24,115 @@ import {
|
|
|
24
24
|
|
|
25
25
|
import {
|
|
26
26
|
arrayLast as y,
|
|
27
|
-
readBufferAsync as
|
|
27
|
+
readBufferAsync as C,
|
|
28
28
|
sleep as c
|
|
29
29
|
} from "./QyUtils.js";
|
|
30
30
|
|
|
31
|
-
class
|
|
32
|
-
constructor(
|
|
33
|
-
|
|
34
|
-
...
|
|
35
|
-
...
|
|
31
|
+
class a {
|
|
32
|
+
constructor(a, t, o) {
|
|
33
|
+
t = {
|
|
34
|
+
...s("QyKVData"),
|
|
35
|
+
...t
|
|
36
36
|
};
|
|
37
|
-
var
|
|
37
|
+
var e = n(a, "snapshot");
|
|
38
38
|
Object.assign(this, {
|
|
39
39
|
qyCache: o,
|
|
40
|
-
options:
|
|
41
|
-
kvFolder:
|
|
42
|
-
snapshotFolder:
|
|
43
|
-
qyKVDataSaver: new p(this,
|
|
44
|
-
qySnapshotCompactor: new i(this,
|
|
45
|
-
qyKVDataCleaner: new r(this,
|
|
40
|
+
options: t,
|
|
41
|
+
kvFolder: a,
|
|
42
|
+
snapshotFolder: e,
|
|
43
|
+
qyKVDataSaver: new p(this, a, t),
|
|
44
|
+
qySnapshotCompactor: new i(this, a, t),
|
|
45
|
+
qyKVDataCleaner: new r(this, a, t),
|
|
46
46
|
compactState: 1,
|
|
47
47
|
compactMaxChangeId: 0,
|
|
48
48
|
snapshotSavedChangeId: 0,
|
|
49
|
-
snapshotInfoNumList: [],
|
|
50
49
|
emptySnapshotNumList: []
|
|
51
50
|
});
|
|
52
51
|
}
|
|
53
|
-
start(
|
|
52
|
+
start(a, t, o, e) {
|
|
54
53
|
var {
|
|
55
54
|
options: n,
|
|
56
|
-
qyKVDataSaver:
|
|
55
|
+
qyKVDataSaver: s,
|
|
57
56
|
qySnapshotCompactor: p,
|
|
58
57
|
qyKVDataCleaner: i
|
|
59
58
|
} = this, {
|
|
60
59
|
maxSnapshotNum: r,
|
|
61
60
|
snapshotMaxChangeId: h
|
|
62
|
-
} = o,
|
|
61
|
+
} = o, s = (s.start(a, r, t, e), p.start(), i.start(), setInterval(() => l(this), n.compactSnapshotsInterval)), r = setInterval(() => (async o => {
|
|
63
62
|
if (!o.unloadingMemory) {
|
|
64
63
|
o.unloadingMemory = !0;
|
|
65
64
|
var {
|
|
66
|
-
qyCache:
|
|
65
|
+
qyCache: e,
|
|
67
66
|
options: n
|
|
68
67
|
} = o;
|
|
69
|
-
let
|
|
70
|
-
if (
|
|
71
|
-
var
|
|
72
|
-
|
|
73
|
-
for (
|
|
74
|
-
|
|
68
|
+
let a = 0, t = 0;
|
|
69
|
+
if (e) {
|
|
70
|
+
var s, p = n.maxFileCountToUnload, n = e.rootDir;
|
|
71
|
+
t = d(o, n), a = t;
|
|
72
|
+
for (s of n.allLoadedSubdirs()) if ((t += d(o, s)) > p && (a += t,
|
|
73
|
+
t = 0, await c(0), o.isStopping)) break;
|
|
75
74
|
}
|
|
76
|
-
|
|
75
|
+
a += t, global.gc && global.gc(), o.unloadingMemory = !1;
|
|
77
76
|
}
|
|
78
77
|
})(this), n.unloadMemInterval);
|
|
79
78
|
return Object.assign(this, {
|
|
80
|
-
maxChangeId:
|
|
79
|
+
maxChangeId: a,
|
|
81
80
|
snapshotSavedChangeId: h,
|
|
82
81
|
qySnapshots: o,
|
|
83
|
-
aclKeyValueMap:
|
|
84
|
-
compactInterval:
|
|
82
|
+
aclKeyValueMap: e,
|
|
83
|
+
compactInterval: s,
|
|
85
84
|
unloadInterval: r
|
|
86
|
-
}), n.bgLoadPosMaps ? (async
|
|
87
|
-
for (var
|
|
88
|
-
if (await async function
|
|
85
|
+
}), n.bgLoadPosMaps ? (async a => {
|
|
86
|
+
for (var t of Object.keys(a.qySnapshots.numToSnapshotMap).sort((a, t) => t - a)) {
|
|
87
|
+
if (await async function s(p, i, r, h, c, l) {
|
|
89
88
|
if (p.isStopping) return void (l && l.close());
|
|
90
|
-
let
|
|
91
|
-
let
|
|
92
|
-
let
|
|
93
|
-
if (!
|
|
89
|
+
let d = p.qySnapshots;
|
|
90
|
+
let m = d.numToSnapshotMap;
|
|
91
|
+
let u = m[i];
|
|
92
|
+
if (!u) return void (l && l.close());
|
|
94
93
|
let {
|
|
95
94
|
info: S,
|
|
96
95
|
prefixPM: f
|
|
97
|
-
} =
|
|
96
|
+
} = u;
|
|
98
97
|
r = r || Object.keys(f).sort().reverse();
|
|
99
98
|
if (0 == r.length) return void (l && l.close());
|
|
100
|
-
if (null == h || h != S.pMVersion) return l && l.close(), c =
|
|
101
|
-
|
|
99
|
+
if (null == h || h != S.pMVersion) return l && l.close(), c = d.getPMFilePath(u),
|
|
100
|
+
s(p, i, r, S.pMVersion, c, await v(c));
|
|
102
101
|
for (;0 < r.length; ) {
|
|
103
|
-
let
|
|
102
|
+
let e = y(r), n = f[e];
|
|
104
103
|
if (n) {
|
|
105
|
-
let [
|
|
106
|
-
if (p.isStopping || !
|
|
107
|
-
if (h != S.pMVersion) return l.close(),
|
|
108
|
-
|
|
104
|
+
let [ a, t ] = n, o = await C(c, l, a, t);
|
|
105
|
+
if (p.isStopping || !m[i]) return void l.close();
|
|
106
|
+
if (h != S.pMVersion) return l.close(), s(p, i, r);
|
|
107
|
+
e in f && d.loadPartialPosMapBuffer(u, e, o);
|
|
109
108
|
}
|
|
110
109
|
r.pop();
|
|
111
110
|
}
|
|
112
111
|
l.close();
|
|
113
112
|
logger.info("bgLoadPosMap:" + i);
|
|
114
|
-
}(
|
|
115
|
-
if (await c(0),
|
|
113
|
+
}(a, t), a.isStopping) break;
|
|
114
|
+
if (await c(0), a.isStopping) break;
|
|
116
115
|
}
|
|
117
|
-
!
|
|
116
|
+
!a.isStopping && a.options.compactAtStart && l(a);
|
|
118
117
|
})(this) : n.compactAtStart && l(this), this;
|
|
119
118
|
}
|
|
120
119
|
async stop() {
|
|
121
120
|
this.isStopping = !0;
|
|
122
121
|
var {
|
|
123
|
-
qyKVDataSaver:
|
|
124
|
-
qySnapshotCompactor:
|
|
122
|
+
qyKVDataSaver: a,
|
|
123
|
+
qySnapshotCompactor: t,
|
|
125
124
|
qySnapshots: o,
|
|
126
|
-
compactInterval:
|
|
125
|
+
compactInterval: e,
|
|
127
126
|
unloadInterval: n,
|
|
128
|
-
qyKVDataCleaner:
|
|
127
|
+
qyKVDataCleaner: s
|
|
129
128
|
} = this;
|
|
130
|
-
clearInterval(
|
|
131
|
-
await
|
|
129
|
+
clearInterval(e), clearInterval(n), await a.stop(), o && o.closeAllFDs(),
|
|
130
|
+
await t.stop(), await s.stop();
|
|
132
131
|
}
|
|
133
|
-
save(
|
|
132
|
+
save(a, t, o, e) {
|
|
134
133
|
var n = this.qyKVDataSaver;
|
|
135
|
-
if (
|
|
136
|
-
n.castSave(
|
|
134
|
+
if (e) return n.callSave(a, t, o);
|
|
135
|
+
n.castSave(a, t, o);
|
|
137
136
|
}
|
|
138
137
|
increaseChangeId() {
|
|
139
138
|
return ++this.maxChangeId;
|
|
@@ -141,183 +140,160 @@ class t {
|
|
|
141
140
|
decreaseChangeId() {
|
|
142
141
|
--this.maxChangeId;
|
|
143
142
|
}
|
|
144
|
-
getValueSync(
|
|
143
|
+
getValueSync(a) {
|
|
145
144
|
var {
|
|
146
|
-
qySnapshots:
|
|
145
|
+
qySnapshots: t,
|
|
147
146
|
aclKeyValueMap: o
|
|
148
147
|
} = this;
|
|
149
|
-
return
|
|
148
|
+
return a in o ? o[a] : t.getValueSync(a);
|
|
150
149
|
}
|
|
151
|
-
removeKey(
|
|
150
|
+
removeKey(a) {
|
|
152
151
|
var {
|
|
153
|
-
qySnapshots:
|
|
152
|
+
qySnapshots: t,
|
|
154
153
|
aclKeyValueMap: o
|
|
155
154
|
} = this;
|
|
156
|
-
if (!(
|
|
157
|
-
o[
|
|
155
|
+
if (!(a in o)) return t.removeKey(a);
|
|
156
|
+
o[a] = void 0;
|
|
158
157
|
}
|
|
159
|
-
onSnapshotSavedChangeId(
|
|
160
|
-
if (this.snapshotSavedChangeId =
|
|
161
|
-
this.compactState = 1),
|
|
162
|
-
var o,
|
|
163
|
-
for (o in
|
|
164
|
-
var
|
|
165
|
-
|
|
158
|
+
onSnapshotSavedChangeId(a, t) {
|
|
159
|
+
if (this.snapshotSavedChangeId = a, 3 == this.compactState && a >= this.compactMaxChangeId && (K(this),
|
|
160
|
+
this.compactState = 1), t) {
|
|
161
|
+
var o, a = t, e = this.qySnapshots, n = e.keyToSnapshotMap;
|
|
162
|
+
for (o in a.posMap) {
|
|
163
|
+
var s = n[o];
|
|
164
|
+
s && e.removeKeyFromSnapshot(s, o);
|
|
166
165
|
}
|
|
167
|
-
|
|
168
|
-
this.qyKVDataSaver.releaseSnapshot(
|
|
166
|
+
e.addMaxSnapshot(a), logger.info("addPMOnlySnapshot: " + e.snapshotMaxChangeId),
|
|
167
|
+
this.qyKVDataSaver.releaseSnapshot(t.snapshotNum);
|
|
169
168
|
}
|
|
170
169
|
}
|
|
171
|
-
onCompactUpdates(
|
|
170
|
+
onCompactUpdates(a) {
|
|
172
171
|
if (2 == this.compactState) {
|
|
173
|
-
var
|
|
174
|
-
pMUpdates:
|
|
172
|
+
var l = this, {
|
|
173
|
+
pMUpdates: a,
|
|
175
174
|
vLUpdates: d,
|
|
176
|
-
combineUpdate:
|
|
177
|
-
} =
|
|
178
|
-
if (
|
|
179
|
-
var a, o =
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
var
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
} =
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
var {
|
|
194
|
-
info: a,
|
|
195
|
-
compactOutedCount: o
|
|
196
|
-
} = t;
|
|
197
|
-
a.outdatedCount += o, t.outdatedCount -= o, t.compactOutedCount = 0;
|
|
198
|
-
})(i), n.push(a);
|
|
199
|
-
}
|
|
175
|
+
combineUpdate: m
|
|
176
|
+
} = a = a;
|
|
177
|
+
if (a) {
|
|
178
|
+
var t = a, o = l.qySnapshots.numToSnapshotMap;
|
|
179
|
+
for (s in t) {
|
|
180
|
+
var e, n = t[s], s = o[s], {
|
|
181
|
+
prefixPM: p,
|
|
182
|
+
info: i
|
|
183
|
+
} = s;
|
|
184
|
+
for (e in p) p[e] = n[e];
|
|
185
|
+
++i.pMVersion, (a => {
|
|
186
|
+
var {
|
|
187
|
+
info: t,
|
|
188
|
+
compactOutedCount: o
|
|
189
|
+
} = a;
|
|
190
|
+
t.outdatedCount += o, a.outdatedCount -= o, a.compactOutedCount = 0;
|
|
191
|
+
})(s);
|
|
200
192
|
}
|
|
201
193
|
}
|
|
202
194
|
if (d) {
|
|
203
|
-
var
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
prefixPM: x,
|
|
217
|
-
info: D
|
|
218
|
-
} = M;
|
|
219
|
-
for (I in K) K[I] = V[I];
|
|
220
|
-
for (q in x) x[q] = L[q];
|
|
221
|
-
++D.pMVersion, ++D.vLVersion, j(M), y.push(m);
|
|
222
|
-
}
|
|
195
|
+
var r = d, h = (a = l).qySnapshots.numToSnapshotMap;
|
|
196
|
+
for (v in r) {
|
|
197
|
+
var c, u, {
|
|
198
|
+
posMap: S,
|
|
199
|
+
prefixPM: f
|
|
200
|
+
} = r[v], v = h[v], {
|
|
201
|
+
posMap: y,
|
|
202
|
+
prefixPM: C,
|
|
203
|
+
info: g
|
|
204
|
+
} = v;
|
|
205
|
+
for (c in y) y[c] = S[c];
|
|
206
|
+
for (u in C) C[u] = f[u];
|
|
207
|
+
++g.pMVersion, ++g.vLVersion, V(v);
|
|
223
208
|
}
|
|
224
209
|
}
|
|
225
|
-
if (
|
|
226
|
-
var
|
|
227
|
-
let {
|
|
228
|
-
qySnapshots: s,
|
|
229
|
-
snapshotInfoNumList: t,
|
|
230
|
-
emptySnapshotNumList: a
|
|
231
|
-
} = u, {
|
|
210
|
+
if (m) {
|
|
211
|
+
var M, d = m;
|
|
212
|
+
let e = l.qySnapshots, {
|
|
232
213
|
numToSnapshotMap: n,
|
|
233
|
-
keyToSnapshotMap:
|
|
234
|
-
} =
|
|
235
|
-
snapshotNum:
|
|
236
|
-
combinedSnapshotNumList:
|
|
237
|
-
posMap:
|
|
238
|
-
} = d,
|
|
239
|
-
posMap:
|
|
240
|
-
info:
|
|
241
|
-
} =
|
|
242
|
-
for (
|
|
243
|
-
|
|
244
|
-
let
|
|
245
|
-
for (let o of
|
|
246
|
-
var
|
|
214
|
+
keyToSnapshotMap: s
|
|
215
|
+
} = e, {
|
|
216
|
+
snapshotNum: a,
|
|
217
|
+
combinedSnapshotNumList: t,
|
|
218
|
+
posMap: p
|
|
219
|
+
} = d, i = n[a], {
|
|
220
|
+
posMap: r,
|
|
221
|
+
info: o
|
|
222
|
+
} = i;
|
|
223
|
+
for (M in r) r[M] = p[M];
|
|
224
|
+
V(i);
|
|
225
|
+
let h = o.totalCount, c = i.outdatedCount;
|
|
226
|
+
for (let o of t) {
|
|
227
|
+
var I, q = n[o];
|
|
247
228
|
let {
|
|
248
|
-
posMap:
|
|
249
|
-
info:
|
|
250
|
-
} =
|
|
251
|
-
for (
|
|
252
|
-
|
|
229
|
+
posMap: a,
|
|
230
|
+
info: t
|
|
231
|
+
} = q;
|
|
232
|
+
for (I in a) r[I] = p[I], s[I] = i;
|
|
233
|
+
V(q), h += t.totalCount, c += q.outdatedCount, e.delSnapshot(q);
|
|
253
234
|
}
|
|
254
|
-
|
|
255
|
-
r.outdatedCount = m, t)).push(o);
|
|
235
|
+
++o.pMVersion, ++o.vLVersion, o.totalCount = h, i.outdatedCount = c;
|
|
256
236
|
}
|
|
257
|
-
this.snapshotSavedChangeId >= this.compactMaxChangeId ? (
|
|
237
|
+
this.snapshotSavedChangeId >= this.compactMaxChangeId ? (K(this), this.compactState = 1) : this.compactState = 3;
|
|
258
238
|
}
|
|
259
239
|
}
|
|
260
240
|
}
|
|
261
241
|
|
|
262
|
-
function
|
|
242
|
+
function V(a) {
|
|
263
243
|
var {
|
|
264
|
-
info:
|
|
244
|
+
info: t,
|
|
265
245
|
compactOutedCount: o
|
|
266
|
-
} =
|
|
267
|
-
|
|
268
|
-
|
|
246
|
+
} = a;
|
|
247
|
+
t.totalCount -= t.outdatedCount + o, t.outdatedCount = 0, a.outdatedCount -= o,
|
|
248
|
+
a.compactOutedCount = 0;
|
|
269
249
|
}
|
|
270
250
|
|
|
271
|
-
function l(
|
|
272
|
-
if (1 ==
|
|
251
|
+
function l(a) {
|
|
252
|
+
if (1 == a.compactState) {
|
|
273
253
|
var {
|
|
274
|
-
maxChangeId:
|
|
254
|
+
maxChangeId: t,
|
|
275
255
|
qySnapshots: o,
|
|
276
|
-
qySnapshotCompactor:
|
|
277
|
-
} =
|
|
256
|
+
qySnapshotCompactor: e
|
|
257
|
+
} = a, {
|
|
278
258
|
emptySnapshotNumList: o,
|
|
279
259
|
pMCompactSnapshotList: n,
|
|
280
|
-
vLCompactSnapshotList:
|
|
260
|
+
vLCompactSnapshotList: s,
|
|
281
261
|
combiningSnapshotList: p
|
|
282
262
|
} = o.calOutdates();
|
|
283
|
-
if (0 < p.length || 0 < n.length || 0 <
|
|
284
|
-
Object.assign(
|
|
263
|
+
if (0 < p.length || 0 < n.length || 0 < s.length) {
|
|
264
|
+
Object.assign(a, {
|
|
285
265
|
compactState: 2,
|
|
286
|
-
compactMaxChangeId:
|
|
266
|
+
compactMaxChangeId: t
|
|
287
267
|
});
|
|
288
|
-
for (var i of [ n,
|
|
289
|
-
|
|
268
|
+
for (var i of [ n, s, p ]) for (var r of i) r.compactOutedCount = r.outdatedCount;
|
|
269
|
+
e.compactSnapshots(n, s, p);
|
|
290
270
|
}
|
|
291
|
-
0 < o.length && (
|
|
271
|
+
0 < o.length && (a.emptySnapshotNumList.push(...o), a.snapshotSavedChangeId >= t ? h(a) : 1 == a.compactState && Object.assign(a, {
|
|
292
272
|
compactState: 3,
|
|
293
|
-
compactMaxChangeId:
|
|
273
|
+
compactMaxChangeId: t
|
|
294
274
|
}));
|
|
295
275
|
}
|
|
296
276
|
}
|
|
297
277
|
|
|
298
|
-
function
|
|
299
|
-
var o =
|
|
300
|
-
let
|
|
301
|
-
if (
|
|
302
|
-
return
|
|
278
|
+
function d(a, t) {
|
|
279
|
+
var o = a.qySnapshots.snapshotMaxChangeId;
|
|
280
|
+
let e = 0;
|
|
281
|
+
if (t.fileMapLoaded) for (var n of t.fileList) ++e, n.cChangeId <= o && (n.visited ? n.visited = !1 : n.unloadFileContent());
|
|
282
|
+
return e;
|
|
303
283
|
}
|
|
304
284
|
|
|
305
|
-
function h(
|
|
285
|
+
function h(a) {
|
|
306
286
|
var {
|
|
307
|
-
qyKVDataCleaner:
|
|
308
|
-
emptySnapshotNumList:
|
|
309
|
-
} =
|
|
310
|
-
0 <
|
|
287
|
+
qyKVDataCleaner: a,
|
|
288
|
+
emptySnapshotNumList: t
|
|
289
|
+
} = a;
|
|
290
|
+
0 < t.length && (a.removeSnapshotsFiles(t), t.length = 0);
|
|
311
291
|
}
|
|
312
292
|
|
|
313
|
-
function
|
|
314
|
-
|
|
315
|
-
qySnapshotCompactor: a,
|
|
316
|
-
snapshotInfoNumList: o
|
|
317
|
-
} = t;
|
|
318
|
-
0 < o.length && (a.saveSnapshotInfos(o), o.length = 0), h(t);
|
|
293
|
+
function K(a) {
|
|
294
|
+
a.qySnapshotCompactor.saveSnapshotInfos(), h(a);
|
|
319
295
|
}
|
|
320
296
|
|
|
321
297
|
export {
|
|
322
|
-
|
|
298
|
+
a as QyKVData
|
|
323
299
|
};
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
import {
|
|
6
6
|
rm as m,
|
|
7
7
|
rename as r,
|
|
8
|
-
readdir as
|
|
8
|
+
readdir as f
|
|
9
9
|
} from "node:fs/promises";
|
|
10
10
|
|
|
11
11
|
import {
|
|
@@ -17,8 +17,8 @@ import {
|
|
|
17
17
|
} from "./QyMessageWorker.js";
|
|
18
18
|
|
|
19
19
|
import {
|
|
20
|
-
folderOrFileExists as
|
|
21
|
-
ensureDir as
|
|
20
|
+
folderOrFileExists as h,
|
|
21
|
+
ensureDir as d
|
|
22
22
|
} from "./QyUtils.js";
|
|
23
23
|
|
|
24
24
|
import {
|
|
@@ -34,7 +34,7 @@ class s extends e {
|
|
|
34
34
|
super(s = {
|
|
35
35
|
...o("QyKVDataCleaner_Worker"),
|
|
36
36
|
...s
|
|
37
|
-
}, [
|
|
37
|
+
}, [], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
|
|
38
38
|
var s = c(e, "history"), a = c(e, "snapshot");
|
|
39
39
|
Object.assign(this, {
|
|
40
40
|
kvFolder: e,
|
|
@@ -49,14 +49,14 @@ class s extends e {
|
|
|
49
49
|
_op_stop() {}
|
|
50
50
|
async _op_cleanAclFiles(e) {
|
|
51
51
|
var s = this, a = e, e = s.aclFolder;
|
|
52
|
-
if (await
|
|
52
|
+
if (await h(e)) {
|
|
53
53
|
var o, r = [];
|
|
54
|
-
for (o of (await
|
|
54
|
+
for (o of (await f(e)).filter(e => /^[1-9]/.test(e))) parseInt(o) <= a && r.push((async (e, s) => {
|
|
55
55
|
var {
|
|
56
56
|
aclFolder: a,
|
|
57
57
|
historyAclFolder: o
|
|
58
58
|
} = e;
|
|
59
|
-
return await
|
|
59
|
+
return await d(o), F(e, s, a, o);
|
|
60
60
|
})(s, o));
|
|
61
61
|
0 < r.length && await Promise.allSettled(r);
|
|
62
62
|
}
|
|
@@ -72,11 +72,12 @@ class s extends e {
|
|
|
72
72
|
}
|
|
73
73
|
async _op_removeSnapshotsFiles(e) {
|
|
74
74
|
let {
|
|
75
|
-
snapshotFolder:
|
|
76
|
-
snapshotInfoFolder:
|
|
75
|
+
snapshotFolder: a,
|
|
76
|
+
snapshotInfoFolder: o
|
|
77
77
|
} = this;
|
|
78
|
-
await Promise.all(e.map(async
|
|
79
|
-
await m(c(
|
|
78
|
+
await Promise.all(e.map(async s => {
|
|
79
|
+
await m(c(o, s + ".json")), await Promise.all([ "pM", "prefixPM", "vL" ].map(e => l(this, s, 1 / 0, e))),
|
|
80
|
+
await m(c(a, "" + s), {
|
|
80
81
|
recursive: !0
|
|
81
82
|
});
|
|
82
83
|
}));
|
|
@@ -96,13 +97,13 @@ async function l(s, a, o, r) {
|
|
|
96
97
|
snapshotFolder: e,
|
|
97
98
|
historySnapshotFolder: t
|
|
98
99
|
} = s, i = c(e, "" + a);
|
|
99
|
-
if (await
|
|
100
|
+
if (await h(i)) {
|
|
100
101
|
let e;
|
|
101
102
|
var l, n = [];
|
|
102
|
-
for (l of await
|
|
103
|
+
for (l of await f(i)) {
|
|
103
104
|
var p = new RegExp("^([0-9]+)_" + r).exec(l);
|
|
104
105
|
p && (p = parseInt(p[1])) < o && (1 == p ? (e || (e = c(t, "" + a),
|
|
105
|
-
await
|
|
106
|
+
await d(e)), n.push(F(s, l, i, e))) : (p = c(i, l), logger.info("Removing " + p),
|
|
106
107
|
n.push(m(p))));
|
|
107
108
|
}
|
|
108
109
|
0 < n.length && await Promise.allSettled(n);
|
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
} from "./QyDefaultOptions.js";
|
|
8
8
|
|
|
9
9
|
import {
|
|
10
|
-
QyAclSaver as
|
|
10
|
+
QyAclSaver as o
|
|
11
11
|
} from "./QyAclSaver.js";
|
|
12
12
|
|
|
13
13
|
import {
|
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
} from "./QyMessageWorker.js";
|
|
16
16
|
|
|
17
17
|
import {
|
|
18
|
-
QySnapshotSaver as
|
|
18
|
+
QySnapshotSaver as n
|
|
19
19
|
} from "./QySnapshotSaver.js";
|
|
20
20
|
|
|
21
21
|
import {
|
|
@@ -33,31 +33,31 @@ class s extends a {
|
|
|
33
33
|
}), Object.assign(this, {
|
|
34
34
|
kvFolder: a,
|
|
35
35
|
aclFolder: t(a, "acl"),
|
|
36
|
-
qyAclSaver: new
|
|
37
|
-
qySnapshotSaver: new
|
|
36
|
+
qyAclSaver: new o(this, s),
|
|
37
|
+
qySnapshotSaver: new n(this, a, s)
|
|
38
38
|
});
|
|
39
39
|
}
|
|
40
|
-
|
|
40
|
+
_op_start(a, s, t, e) {
|
|
41
41
|
var {
|
|
42
|
-
qyAclSaver:
|
|
43
|
-
qySnapshotSaver:
|
|
42
|
+
qyAclSaver: o,
|
|
43
|
+
qySnapshotSaver: n
|
|
44
44
|
} = this;
|
|
45
45
|
Object.assign(this, {
|
|
46
46
|
maxChangeId: a,
|
|
47
47
|
snapshotMaxChangeId: a,
|
|
48
48
|
currentAclNum: (t || s) + 1
|
|
49
|
-
}),
|
|
49
|
+
}), o.start(u(this)), n.start(a, t, e);
|
|
50
50
|
}
|
|
51
|
-
async
|
|
51
|
+
async _op_stop() {
|
|
52
52
|
var {
|
|
53
53
|
qyAclSaver: a,
|
|
54
54
|
options: s,
|
|
55
55
|
qySnapshotSaver: t,
|
|
56
56
|
currentAclNum: e,
|
|
57
|
-
maxChangeId:
|
|
58
|
-
snapshotMaxChangeId:
|
|
57
|
+
maxChangeId: o,
|
|
58
|
+
snapshotMaxChangeId: n
|
|
59
59
|
} = this;
|
|
60
|
-
this.isStopping = !0, await a.stop(), s.saveSnapshotAtStop &&
|
|
60
|
+
this.isStopping = !0, await a.stop(), s.saveSnapshotAtStop && n < o && (this.isSavingSnapshot && await new Promise(a => this.saveSnapshotResolve = a),
|
|
61
61
|
logger.log(`Saving snapshot ${e} at stop.`), t.saveSnapshot(e)), await t.stop();
|
|
62
62
|
}
|
|
63
63
|
callSave(a, s, t) {
|
|
@@ -74,9 +74,9 @@ class s extends a {
|
|
|
74
74
|
}
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
-
function r(a, s, t, e,
|
|
77
|
+
function r(a, s, t, e, o) {
|
|
78
78
|
var {
|
|
79
|
-
qyAclSaver:
|
|
79
|
+
qyAclSaver: n,
|
|
80
80
|
qySnapshotSaver: r,
|
|
81
81
|
options: h,
|
|
82
82
|
snapshotMaxChangeId: i,
|
|
@@ -84,14 +84,14 @@ function r(a, s, t, e, n) {
|
|
|
84
84
|
isSavingSnapshot: S
|
|
85
85
|
} = a;
|
|
86
86
|
let v;
|
|
87
|
-
|
|
87
|
+
o ? v = n.callSave(s, t) : n.castSave(s, t);
|
|
88
88
|
var l = t, c = e;
|
|
89
89
|
if (c) {
|
|
90
90
|
var g = c.length;
|
|
91
91
|
for (let a = 0; a < g; a += 2) m(l, c[a]);
|
|
92
92
|
}
|
|
93
93
|
return r.castSave(s, t, e), a.maxChangeId = s, !S && s - i >= h.snapshotMaxChangeCount && (++a.currentAclNum,
|
|
94
|
-
|
|
94
|
+
n.switch(u(a)), r.saveSnapshot(p), Object.assign(a, {
|
|
95
95
|
snapshotMaxChangeId: s,
|
|
96
96
|
isSavingSnapshot: !0
|
|
97
97
|
})), v;
|
package/src/QyLogFileSaver.js
CHANGED