datafolder 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -31
- package/index.js +1 -3
- package/package.json +5 -3
- package/sample.js +37 -37
- package/scripts/airbnb.js +10 -11
- package/scripts/airbnb2.js +10 -11
- package/scripts/airbnb2_queries.js +20 -22
- package/scripts/airbnb_queries.js +23 -22
- package/scripts/downloadFile.js +29 -31
- package/scripts/express.js +7 -5
- package/scripts/fastify.js +12 -11
- package/scripts/flights.js +9 -9
- package/scripts/flights2.js +9 -9
- package/scripts/flights2_queries.js +107 -105
- package/scripts/flights_queries.js +113 -109
- package/src/QyAcl.js +153 -152
- package/src/QyAclSaver.js +16 -10
- package/src/QyAclSaver_Worker.js +21 -15
- package/src/QyAclWriter.js +23 -13
- package/src/QyBatch.js +298 -263
- package/src/QyBinWriter.js +49 -37
- package/src/QyCD.js +13 -14
- package/src/QyCache.js +177 -140
- package/src/QyCompressor.js +71 -45
- package/src/QyCompressor_Worker.js +17 -9
- package/src/QyDB.js +67 -12
- package/src/QyDefaultOptions.js +13 -10
- package/src/QyDir.js +89 -53
- package/src/QyFetch.js +27 -24
- package/src/QyFile.js +89 -77
- package/src/QyFileContentUtils.js +129 -132
- package/src/QyFileLogger.js +63 -41
- package/src/QyIndex.js +175 -197
- package/src/QyIndexPropTree.js +25 -25
- package/src/QyIntersectNode.js +34 -31
- package/src/QyItem.js +54 -52
- package/src/QyJsonSaver.js +12 -6
- package/src/QyKVData.js +279 -245
- package/src/QyKVDataCleaner.js +12 -6
- package/src/QyKVDataCleaner_Worker.js +77 -59
- package/src/QyKVDataLoader.js +67 -46
- package/src/QyKVDataSaver.js +15 -9
- package/src/QyKVDataSaver_Worker.js +83 -60
- package/src/QyLogFileSaver.js +12 -6
- package/src/QyLogFileSaver_Worker.js +24 -14
- package/src/QyLogger.js +51 -40
- package/src/QyMessageWorker.js +43 -29
- package/src/QyMessager.js +74 -54
- package/src/QyMessager_Run.js +20 -7
- package/src/QyMod.js +15 -11
- package/src/QyOpRunner.js +58 -41
- package/src/QyQueue.js +35 -30
- package/src/QySaver.js +52 -29
- package/src/QySnapshotCompactor.js +15 -9
- package/src/QySnapshotCompactor_Worker.js +131 -110
- package/src/QySnapshotSaver.js +15 -9
- package/src/QySnapshotSaver_Worker.js +125 -110
- package/src/QySnapshots.js +264 -242
- package/src/QyStructure.js +35 -27
- package/src/QyTrigger.js +62 -55
- package/src/QyUnionNode.js +54 -56
- package/src/QyUtils.js +170 -158
- package/src/QyWorker.js +34 -20
- package/src/hackernews.js +63 -71
- package/src/rpc/getFileContent.js +7 -7
- package/src/rpc/getSubItems.js +12 -28
- package/src/rpc/queryFiles.js +2 -25
- package/src/simulate.js +85 -80
- package/src/simulate2.js +77 -78
- package/src/soc_pokec.js +44 -32
- package/static/datafolderui/QyStyledElement.js +12 -12
- package/static/datafolderui/QyUIFolderTree.js +20 -20
- package/static/datafolderui/QyUIUtils.js +8 -8
- package/static/datafolderui/explore-tree.js +31 -31
- package/static/datafolderui/folder-tree.js +2 -2
- package/static/datafolderui/hscrpt.js +9 -17
- package/static/datafolderui/json-view.js +3 -5
- package/static/datafolderui/kv-input.js +15 -15
- package/static/datafolderui/list-input.js +17 -17
- package/static/datafolderui/main-page.js +18 -18
- package/static/datafolderui/num-input.js +5 -5
- package/static/datafolderui/query-editor.js +6 -1
- package/static/datafolderui/querySelectorShadowRoot.js +15 -19
- package/static/datafolderui/resizable-input.js +18 -18
|
@@ -1,117 +1,65 @@
|
|
|
1
|
-
|
|
1
|
+
import {
|
|
2
|
+
join as e
|
|
3
|
+
} from "node:path";
|
|
2
4
|
|
|
3
|
-
|
|
5
|
+
import {
|
|
6
|
+
getDefaultOptions as o
|
|
7
|
+
} from "./QyDefaultOptions.js";
|
|
8
|
+
|
|
9
|
+
import {
|
|
10
|
+
QyMessageWorker as a
|
|
11
|
+
} from "./QyMessageWorker.js";
|
|
12
|
+
|
|
13
|
+
import {
|
|
14
|
+
QySnapshots as n
|
|
15
|
+
} from "./QySnapshots.js";
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
QyKVDataCleaner as p
|
|
19
|
+
} from "./QyKVDataCleaner.js";
|
|
20
|
+
|
|
21
|
+
class s extends a {
|
|
4
22
|
constructor({
|
|
5
23
|
kvFolder: a,
|
|
6
24
|
options: s
|
|
7
25
|
}) {
|
|
8
26
|
super(s = {
|
|
9
|
-
...
|
|
27
|
+
...o("QySnapshotSaver_Worker"),
|
|
10
28
|
...s
|
|
11
29
|
}, [ "stop", "waitForSnapshotSaved" ]);
|
|
12
|
-
var
|
|
30
|
+
var t = e(a, "snapshot");
|
|
13
31
|
Object.assign(this, {
|
|
14
32
|
kvFolder: a,
|
|
15
|
-
snapshotFolder:
|
|
16
|
-
qySnapshots: new
|
|
17
|
-
qyKVDataCleaner: new
|
|
33
|
+
snapshotFolder: t,
|
|
34
|
+
qySnapshots: new n(t, s),
|
|
35
|
+
qyKVDataCleaner: new p(this, a, s),
|
|
18
36
|
aclKeyValueMap: {}
|
|
19
37
|
});
|
|
20
38
|
}
|
|
21
|
-
start(a, s,
|
|
22
|
-
this.maxChangeId = a, this.qyKVDataCleaner.start(), s && this.waitForSnapshotSaved(this
|
|
39
|
+
start(a, s, t) {
|
|
40
|
+
this.maxChangeId = a, this.qyKVDataCleaner.start(), s && this.waitForSnapshotSaved(h(this, s, a, t));
|
|
23
41
|
}
|
|
24
|
-
castSave(a, s,
|
|
42
|
+
castSave(a, s, t) {
|
|
25
43
|
var {
|
|
26
|
-
aclKeyValueMap:
|
|
27
|
-
qySnapshots:
|
|
44
|
+
aclKeyValueMap: e,
|
|
45
|
+
qySnapshots: o
|
|
28
46
|
} = this;
|
|
29
|
-
|
|
47
|
+
o.applyCmdObj(s, e, t), this.maxChangeId = a;
|
|
30
48
|
}
|
|
31
49
|
async saveSnapshot(a) {
|
|
32
50
|
var {
|
|
33
51
|
maxChangeId: s,
|
|
34
|
-
aclKeyValueMap:
|
|
52
|
+
aclKeyValueMap: t
|
|
35
53
|
} = this;
|
|
36
|
-
this.aclKeyValueMap = {}, this.waitForSnapshotSaved(this
|
|
54
|
+
this.aclKeyValueMap = {}, this.waitForSnapshotSaved(h(this, a, s, t));
|
|
37
55
|
}
|
|
38
56
|
releaseSnapshot(a) {
|
|
39
|
-
var s,
|
|
40
|
-
numToSnapshotMap:
|
|
41
|
-
keyToSnapshotMap:
|
|
42
|
-
} =
|
|
43
|
-
for (s in delete
|
|
44
|
-
|
|
45
|
-
}
|
|
46
|
-
_createMaxSnapshot(a, s, e) {
|
|
47
|
-
var t = this._processOutdates(), a = {
|
|
48
|
-
snapshotNum: a,
|
|
49
|
-
keyValueMap: e,
|
|
50
|
-
outdatedCount: 0,
|
|
51
|
-
info: {
|
|
52
|
-
maxChangeId: s,
|
|
53
|
-
pMVersion: 0,
|
|
54
|
-
vLVersion: 0
|
|
55
|
-
}
|
|
56
|
-
}, e = (this.qySnapshots.addMaxSnapshot(a), this._saveKeyValueMapAndSnapshotInfo(a));
|
|
57
|
-
return {
|
|
58
|
-
saveMaxSnapshotInfoPromise: e,
|
|
59
|
-
outdates: t
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
async _saveKeyValueMapAndSnapshotInfo(a) {
|
|
63
|
-
var s = this.qySnapshots, {
|
|
64
|
-
snapshotNum: a,
|
|
65
|
-
info: e
|
|
66
|
-
} = (await s.saveKeyValueMap(a), a);
|
|
67
|
-
return await s.saveSnapshotInfo(a, e), logger.log("Created snapshot " + a),
|
|
68
|
-
{
|
|
69
|
-
snapshotNum: a,
|
|
70
|
-
maxChangeId: e.maxChangeId
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
_processOutdates() {
|
|
74
|
-
var a, s = this.qySnapshots, {
|
|
75
|
-
emptySnapshotNumList: e,
|
|
76
|
-
combiningSnapshotList: t,
|
|
77
|
-
pMCompactSnapshotList: n
|
|
78
|
-
} = s.calOutdates(), o = {}, p = [], h = [], r = [];
|
|
79
|
-
for (a of n) {
|
|
80
|
-
var {
|
|
81
|
-
snapshotNum: i,
|
|
82
|
-
info: l
|
|
83
|
-
} = a;
|
|
84
|
-
p.push(s.savePosMap(a)), h.push([ i, l.pMVersion ]), o[i] = {
|
|
85
|
-
...l
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
n = t.length;
|
|
89
|
-
if (0 < n) {
|
|
90
|
-
var u, S, y = s.keyToSnapshotMap, {
|
|
91
|
-
keyValueMap: M,
|
|
92
|
-
info: n,
|
|
93
|
-
snapshotNum: d
|
|
94
|
-
} = u = s.getMaxCombinedSnapshot(t);
|
|
95
|
-
for (S of t) if (u != S) {
|
|
96
|
-
var g, {
|
|
97
|
-
keyValueMap: v,
|
|
98
|
-
snapshotNum: m
|
|
99
|
-
} = S;
|
|
100
|
-
for (g in v) M[g] = v[g], y[g] = u;
|
|
101
|
-
s.delSnapshot(S), e.push(m);
|
|
102
|
-
}
|
|
103
|
-
p.push(s.saveKeyValueMap(u)), h.push([ d, n.pMVersion ]), r.push(d, n.vLVersion),
|
|
104
|
-
o[d] = {
|
|
105
|
-
...n
|
|
106
|
-
};
|
|
107
|
-
}
|
|
108
|
-
return {
|
|
109
|
-
emptySnapshotNumList: e,
|
|
110
|
-
pMChangeList: h,
|
|
111
|
-
vLChange: r,
|
|
112
|
-
snapshotInfoMap: o,
|
|
113
|
-
promises: p
|
|
114
|
-
};
|
|
57
|
+
var s, t = this.qySnapshots, {
|
|
58
|
+
numToSnapshotMap: e,
|
|
59
|
+
keyToSnapshotMap: o
|
|
60
|
+
} = t, e = e[a], a = e.keyValueMap;
|
|
61
|
+
for (s in delete e.keyValueMap, a) delete o[s];
|
|
62
|
+
t.delSnapshot(e), global.gc && global.gc();
|
|
115
63
|
}
|
|
116
64
|
async _op_stop() {
|
|
117
65
|
var {
|
|
@@ -124,13 +72,13 @@ class QySnapshotSaver_Worker extends QyMessageWorker {
|
|
|
124
72
|
saveMaxSnapshotInfoPromise: a,
|
|
125
73
|
outdates: s
|
|
126
74
|
}) {
|
|
127
|
-
var
|
|
75
|
+
var t, {
|
|
128
76
|
snapshotNum: a,
|
|
129
|
-
maxChangeId:
|
|
77
|
+
maxChangeId: e
|
|
130
78
|
} = await a, {
|
|
131
79
|
emptySnapshotNumList: s,
|
|
132
|
-
pMChangeList:
|
|
133
|
-
vLChange:
|
|
80
|
+
pMChangeList: o,
|
|
81
|
+
vLChange: n,
|
|
134
82
|
snapshotInfoMap: p,
|
|
135
83
|
promises: h
|
|
136
84
|
} = s, {
|
|
@@ -138,27 +86,94 @@ class QySnapshotSaver_Worker extends QyMessageWorker {
|
|
|
138
86
|
qySnapshots: i,
|
|
139
87
|
options: l
|
|
140
88
|
} = this;
|
|
141
|
-
for (
|
|
142
|
-
p) h.push(i.saveSnapshotInfo(
|
|
89
|
+
for (t in r.cleanAclFiles(a), 0 < h.length && await Promise.all(h), h.length = 0,
|
|
90
|
+
p) h.push(i.saveSnapshotInfo(t, p[t]));
|
|
143
91
|
0 < h.length && await Promise.all(h), 0 < s.length && r.removeSnapshotsFiles(s),
|
|
144
|
-
0 <
|
|
92
|
+
0 < o.length && r.cleanPMFiles(o), 0 < n.length && r.cleanVLFiles(n[0], n[1]);
|
|
145
93
|
let u;
|
|
146
94
|
a = i.snapshotCount;
|
|
147
|
-
0 < a && a >= l.maxInMemSnapshotCount && (delete (s =
|
|
148
|
-
|
|
95
|
+
0 < a && a >= l.maxInMemSnapshotCount && (delete (s = (a => {
|
|
96
|
+
let s = a.qySnapshots.numToSnapshotMap, t;
|
|
97
|
+
for (var e in s) {
|
|
98
|
+
var o = s[e];
|
|
99
|
+
(!t || t.snapshotNum > e) && (t = o);
|
|
100
|
+
}
|
|
101
|
+
return t;
|
|
102
|
+
})(this)).prefixPM, delete (u = {
|
|
149
103
|
...s
|
|
150
104
|
}).keyValueMap, delete s.posMap, s.isOut = !0, logger.info("Sending snapshot " + s.snapshotNum)),
|
|
151
|
-
this.castParent("onSnapshotSavedChangeId",
|
|
105
|
+
this.castParent("onSnapshotSavedChangeId", e, u);
|
|
152
106
|
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function h(a, s, t, e) {
|
|
110
|
+
var o = (a => {
|
|
111
|
+
var s, t = a.qySnapshots, {
|
|
112
|
+
emptySnapshotNumList: e,
|
|
113
|
+
combiningSnapshotList: a,
|
|
114
|
+
pMCompactSnapshotList: o
|
|
115
|
+
} = t.calOutdates(), n = {}, p = [], h = [], r = [];
|
|
116
|
+
for (s of o) {
|
|
117
|
+
var {
|
|
118
|
+
snapshotNum: i,
|
|
119
|
+
info: l
|
|
120
|
+
} = s;
|
|
121
|
+
p.push(t.savePosMap(s)), h.push([ i, l.pMVersion ]), n[i] = {
|
|
122
|
+
...l
|
|
123
|
+
};
|
|
159
124
|
}
|
|
160
|
-
|
|
161
|
-
|
|
125
|
+
let u = a.length, m;
|
|
126
|
+
if (0 < u) {
|
|
127
|
+
var S, y = t.keyToSnapshotMap, {
|
|
128
|
+
keyValueMap: d,
|
|
129
|
+
info: o,
|
|
130
|
+
snapshotNum: v
|
|
131
|
+
} = m = t.getMaxCombinedSnapshot(a);
|
|
132
|
+
for (S of a) if (m != S) {
|
|
133
|
+
var g, {
|
|
134
|
+
keyValueMap: M,
|
|
135
|
+
snapshotNum: f
|
|
136
|
+
} = S;
|
|
137
|
+
for (g in M) d[g] = M[g], y[g] = m;
|
|
138
|
+
t.delSnapshot(S), e.push(f);
|
|
139
|
+
}
|
|
140
|
+
p.push(t.saveKeyValueMap(m)), h.push([ v, o.pMVersion ]), r.push(v, o.vLVersion),
|
|
141
|
+
n[v] = {
|
|
142
|
+
...o
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
emptySnapshotNumList: e,
|
|
147
|
+
pMChangeList: h,
|
|
148
|
+
vLChange: r,
|
|
149
|
+
snapshotInfoMap: n,
|
|
150
|
+
promises: p
|
|
151
|
+
};
|
|
152
|
+
})(a), s = {
|
|
153
|
+
snapshotNum: s,
|
|
154
|
+
keyValueMap: e,
|
|
155
|
+
outdatedCount: 0,
|
|
156
|
+
info: {
|
|
157
|
+
maxChangeId: t,
|
|
158
|
+
pMVersion: 0,
|
|
159
|
+
vLVersion: 0
|
|
160
|
+
}
|
|
161
|
+
}, e = (a.qySnapshots.addMaxSnapshot(s), (async (a, s) => {
|
|
162
|
+
await (a = a.qySnapshots).saveKeyValueMap(s);
|
|
163
|
+
var {
|
|
164
|
+
snapshotNum: s,
|
|
165
|
+
info: t
|
|
166
|
+
} = s;
|
|
167
|
+
return await a.saveSnapshotInfo(s, t), logger.log("Created snapshot " + s),
|
|
168
|
+
{
|
|
169
|
+
snapshotNum: s,
|
|
170
|
+
maxChangeId: t.maxChangeId
|
|
171
|
+
};
|
|
172
|
+
})(a, s));
|
|
173
|
+
return {
|
|
174
|
+
saveMaxSnapshotInfoPromise: e,
|
|
175
|
+
outdates: o
|
|
176
|
+
};
|
|
162
177
|
}
|
|
163
178
|
|
|
164
|
-
|
|
179
|
+
export default s;
|