datafolder 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -31
- package/index.js +1 -3
- package/package.json +5 -3
- package/sample.js +37 -37
- package/scripts/airbnb.js +10 -11
- package/scripts/airbnb2.js +10 -11
- package/scripts/airbnb2_queries.js +20 -22
- package/scripts/airbnb_queries.js +23 -22
- package/scripts/downloadFile.js +29 -31
- package/scripts/express.js +7 -5
- package/scripts/fastify.js +12 -11
- package/scripts/flights.js +9 -9
- package/scripts/flights2.js +9 -9
- package/scripts/flights2_queries.js +107 -105
- package/scripts/flights_queries.js +113 -109
- package/src/QyAcl.js +153 -152
- package/src/QyAclSaver.js +16 -10
- package/src/QyAclSaver_Worker.js +21 -15
- package/src/QyAclWriter.js +23 -13
- package/src/QyBatch.js +298 -263
- package/src/QyBinWriter.js +49 -37
- package/src/QyCD.js +13 -14
- package/src/QyCache.js +177 -140
- package/src/QyCompressor.js +71 -45
- package/src/QyCompressor_Worker.js +17 -9
- package/src/QyDB.js +67 -12
- package/src/QyDefaultOptions.js +13 -10
- package/src/QyDir.js +89 -53
- package/src/QyFetch.js +27 -24
- package/src/QyFile.js +89 -77
- package/src/QyFileContentUtils.js +129 -132
- package/src/QyFileLogger.js +63 -41
- package/src/QyIndex.js +175 -197
- package/src/QyIndexPropTree.js +25 -25
- package/src/QyIntersectNode.js +34 -31
- package/src/QyItem.js +54 -52
- package/src/QyJsonSaver.js +12 -6
- package/src/QyKVData.js +279 -245
- package/src/QyKVDataCleaner.js +12 -6
- package/src/QyKVDataCleaner_Worker.js +77 -59
- package/src/QyKVDataLoader.js +67 -46
- package/src/QyKVDataSaver.js +15 -9
- package/src/QyKVDataSaver_Worker.js +83 -60
- package/src/QyLogFileSaver.js +12 -6
- package/src/QyLogFileSaver_Worker.js +24 -14
- package/src/QyLogger.js +51 -40
- package/src/QyMessageWorker.js +43 -29
- package/src/QyMessager.js +74 -54
- package/src/QyMessager_Run.js +20 -7
- package/src/QyMod.js +15 -11
- package/src/QyOpRunner.js +58 -41
- package/src/QyQueue.js +35 -30
- package/src/QySaver.js +52 -29
- package/src/QySnapshotCompactor.js +15 -9
- package/src/QySnapshotCompactor_Worker.js +131 -110
- package/src/QySnapshotSaver.js +15 -9
- package/src/QySnapshotSaver_Worker.js +125 -110
- package/src/QySnapshots.js +264 -242
- package/src/QyStructure.js +35 -27
- package/src/QyTrigger.js +62 -55
- package/src/QyUnionNode.js +54 -56
- package/src/QyUtils.js +170 -158
- package/src/QyWorker.js +34 -20
- package/src/hackernews.js +63 -71
- package/src/rpc/getFileContent.js +7 -7
- package/src/rpc/getSubItems.js +12 -28
- package/src/rpc/queryFiles.js +2 -25
- package/src/simulate.js +85 -80
- package/src/simulate2.js +77 -78
- package/src/soc_pokec.js +44 -32
- package/static/datafolderui/QyStyledElement.js +12 -12
- package/static/datafolderui/QyUIFolderTree.js +20 -20
- package/static/datafolderui/QyUIUtils.js +8 -8
- package/static/datafolderui/explore-tree.js +31 -31
- package/static/datafolderui/folder-tree.js +2 -2
- package/static/datafolderui/hscrpt.js +9 -17
- package/static/datafolderui/json-view.js +3 -5
- package/static/datafolderui/kv-input.js +15 -15
- package/static/datafolderui/list-input.js +17 -17
- package/static/datafolderui/main-page.js +18 -18
- package/static/datafolderui/num-input.js +5 -5
- package/static/datafolderui/query-editor.js +6 -1
- package/static/datafolderui/querySelectorShadowRoot.js +15 -19
- package/static/datafolderui/resizable-input.js +18 -18
package/src/QyStructure.js
CHANGED
|
@@ -1,50 +1,58 @@
|
|
|
1
|
-
|
|
2
|
-
readdir,
|
|
3
|
-
rm,
|
|
4
|
-
writeFile,
|
|
5
|
-
rename
|
|
6
|
-
}
|
|
1
|
+
import {
|
|
2
|
+
readdir as w,
|
|
3
|
+
rm as v,
|
|
4
|
+
writeFile as g,
|
|
5
|
+
rename as F
|
|
6
|
+
} from "node:fs/promises";
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
import {
|
|
9
|
+
join as x
|
|
10
|
+
} from "node:path";
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
ensureDir as y
|
|
14
|
+
} from "./QyUtils.js";
|
|
15
|
+
|
|
16
|
+
async function P(e, r) {
|
|
9
17
|
let t = {};
|
|
10
18
|
for (var i of r.subdirList) {
|
|
11
19
|
let e = i.name;
|
|
12
20
|
for (;r.getFile(e); ) e += "_d";
|
|
13
|
-
e =
|
|
21
|
+
e = _(e), t[e] = {
|
|
14
22
|
x: i,
|
|
15
23
|
count: 0
|
|
16
24
|
};
|
|
17
25
|
}
|
|
18
|
-
await
|
|
19
|
-
var
|
|
20
|
-
let
|
|
21
|
-
for (
|
|
26
|
+
await y(e);
|
|
27
|
+
var o, a = [], s = [];
|
|
28
|
+
let n = {};
|
|
29
|
+
for (o of await w(e, {
|
|
22
30
|
withFileTypes: !0
|
|
23
31
|
})) {
|
|
24
|
-
var
|
|
25
|
-
|
|
32
|
+
var u = o.name;
|
|
33
|
+
o.isFile() ? a.push(u) : (s.push(u), n[u] = 1);
|
|
26
34
|
}
|
|
27
|
-
var l, p, m, c, h
|
|
28
|
-
for (
|
|
29
|
-
t[
|
|
35
|
+
var f, l, p, m, c, h = [];
|
|
36
|
+
for (f of s) t[f] || ((l = d((l = f) + "_d") || d(l.replace(/_d$/, ""))) ? (h.push(F(x(e, f), x(e, l))),
|
|
37
|
+
t[l].count = 1) : h.push(v(x(e, f), {
|
|
30
38
|
recursive: !0
|
|
31
39
|
})));
|
|
32
|
-
for (
|
|
33
|
-
await Promise.all(
|
|
34
|
-
for (
|
|
40
|
+
for (p of a) r.getFile(p) || h.push(v(x(e, p)));
|
|
41
|
+
await Promise.all(h), h.length = 0;
|
|
42
|
+
for (m of r.fileList) h.push(g(x(e, _(m.name)), m.contentAsText, {
|
|
35
43
|
flush: !0
|
|
36
44
|
}));
|
|
37
|
-
for (
|
|
45
|
+
for (c in await Promise.all(h), h.length = 0, t) h.push(P(x(e, c), t[c].x));
|
|
38
46
|
function d(e) {
|
|
39
|
-
return e && 0 == t[e]?.count && !
|
|
47
|
+
return e && 0 == t[e]?.count && !n[e] ? e : void 0;
|
|
40
48
|
}
|
|
41
|
-
await Promise.all(
|
|
49
|
+
await Promise.all(h);
|
|
42
50
|
}
|
|
43
51
|
|
|
44
|
-
function
|
|
52
|
+
function _(e) {
|
|
45
53
|
return e.replace(/"/g, "");
|
|
46
54
|
}
|
|
47
55
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
}
|
|
56
|
+
export {
|
|
57
|
+
P as dumpStructure
|
|
58
|
+
};
|
package/src/QyTrigger.js
CHANGED
|
@@ -1,15 +1,23 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
} = require("./QyFileContentUtils.js"), {
|
|
5
|
-
isEmptyObj,
|
|
6
|
-
isMatchName,
|
|
7
|
-
getMatchParam,
|
|
8
|
-
arrayLast,
|
|
9
|
-
propNameJoin
|
|
10
|
-
} = require("./QyUtils.js"), isArray = Array.isArray;
|
|
1
|
+
import {
|
|
2
|
+
isDeepStrictEqual as p
|
|
3
|
+
} from "node:util";
|
|
11
4
|
|
|
12
|
-
|
|
5
|
+
import {
|
|
6
|
+
getValByNamePath as c,
|
|
7
|
+
setValByNamePath as u
|
|
8
|
+
} from "./QyFileContentUtils.js";
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
isEmptyObj as e,
|
|
12
|
+
isMatchName as l,
|
|
13
|
+
getMatchParam as d,
|
|
14
|
+
arrayLast as f,
|
|
15
|
+
propNameJoin as N
|
|
16
|
+
} from "./QyUtils.js";
|
|
17
|
+
|
|
18
|
+
let v = Array.isArray;
|
|
19
|
+
|
|
20
|
+
class h {
|
|
13
21
|
constructor(e, r) {
|
|
14
22
|
Object.assign(this, {
|
|
15
23
|
name: e,
|
|
@@ -20,27 +28,27 @@ class QyTriggerNode {
|
|
|
20
28
|
}
|
|
21
29
|
insertTrigger(e, r, t, a) {
|
|
22
30
|
let i = this;
|
|
23
|
-
for (var
|
|
24
|
-
|
|
25
|
-
var
|
|
26
|
-
i = s
|
|
31
|
+
for (var s of r) {
|
|
32
|
+
l(s) && (s = "*");
|
|
33
|
+
var o = i.children;
|
|
34
|
+
i = o[s] || (o[s] = new h(s, i));
|
|
27
35
|
}
|
|
28
|
-
var
|
|
29
|
-
|
|
36
|
+
var n = i.triggerMap, g = n[e];
|
|
37
|
+
g ? Object.assign(g, {
|
|
30
38
|
triggerName: e,
|
|
31
39
|
fileNamePath: r,
|
|
32
40
|
propNamePathList: t,
|
|
33
41
|
fun: a
|
|
34
|
-
}) :
|
|
42
|
+
}) : n[e] = new m(e, r, t, a);
|
|
35
43
|
}
|
|
36
44
|
removeTrigger(e, r) {
|
|
37
45
|
let t = this;
|
|
38
|
-
for (var a of r) if (
|
|
46
|
+
for (var a of r) if (l(a) && (a = "*"), !(t = t.children[a])) return !1;
|
|
39
47
|
r = t.triggerMap;
|
|
40
48
|
return !!r[e] && (delete r[e], t.isEmpty && t.removeEmptyNodes(), !0);
|
|
41
49
|
}
|
|
42
50
|
get isEmpty() {
|
|
43
|
-
return
|
|
51
|
+
return e(this.children) && e(this.triggerMap);
|
|
44
52
|
}
|
|
45
53
|
removeEmptyNodes() {
|
|
46
54
|
let e = this;
|
|
@@ -60,7 +68,7 @@ class QyTriggerNode {
|
|
|
60
68
|
}
|
|
61
69
|
}
|
|
62
70
|
|
|
63
|
-
class
|
|
71
|
+
class m {
|
|
64
72
|
constructor(e, r, t, a) {
|
|
65
73
|
Object.assign(this, {
|
|
66
74
|
triggerName: e,
|
|
@@ -73,61 +81,60 @@ class QyTrigger {
|
|
|
73
81
|
var t, {
|
|
74
82
|
triggerName: a,
|
|
75
83
|
fileNamePath: i,
|
|
76
|
-
propNamePathList:
|
|
77
|
-
fun:
|
|
78
|
-
} = this,
|
|
79
|
-
let
|
|
80
|
-
for (t of
|
|
81
|
-
var
|
|
82
|
-
|
|
83
|
-
oldValue:
|
|
84
|
-
newValue:
|
|
85
|
-
}, (
|
|
86
|
-
(o = o || {
|
|
84
|
+
propNamePathList: s,
|
|
85
|
+
fun: o
|
|
86
|
+
} = this, n = e.fileContent;
|
|
87
|
+
let g;
|
|
88
|
+
for (t of s) {
|
|
89
|
+
var l = c(r.oldValue, t), f = c(r.newValue, t);
|
|
90
|
+
p(l, f) || (l = {
|
|
91
|
+
oldValue: l,
|
|
92
|
+
newValue: f
|
|
93
|
+
}, (v(f) || v(c(n, t))) && (l.isArrayDelta = !0), (g = g || {
|
|
87
94
|
_file: e,
|
|
88
|
-
_fileContent:
|
|
89
|
-
})[
|
|
95
|
+
_fileContent: n
|
|
96
|
+
})[N(t)] = l, u(g, t, l));
|
|
90
97
|
}
|
|
91
|
-
if (
|
|
98
|
+
if (g) {
|
|
92
99
|
var {
|
|
93
|
-
namePath:
|
|
94
|
-
qyCache:
|
|
100
|
+
namePath: h,
|
|
101
|
+
qyCache: s
|
|
95
102
|
} = e;
|
|
96
|
-
for (let e = 0; e <
|
|
97
|
-
var
|
|
98
|
-
|
|
103
|
+
for (let e = 0; e < h.length; ++e) {
|
|
104
|
+
var m = d(i[e]);
|
|
105
|
+
m && (g[m] = h[e]);
|
|
99
106
|
}
|
|
100
107
|
try {
|
|
101
|
-
|
|
108
|
+
o && o(g), s.emit(a, g);
|
|
102
109
|
} catch (e) {
|
|
103
|
-
logger.error(`Running trigger ${a} failed:`,
|
|
110
|
+
logger.error(`Running trigger ${a} failed:`, g, e);
|
|
104
111
|
}
|
|
105
112
|
}
|
|
106
113
|
}
|
|
107
114
|
}
|
|
108
115
|
|
|
109
|
-
let
|
|
116
|
+
let t = [];
|
|
110
117
|
|
|
111
|
-
function
|
|
112
|
-
return e ?
|
|
118
|
+
function y(e, r) {
|
|
119
|
+
return e ? l(r) ? Object.values(e) : (e = e[r]) ? [ e ] : t : t;
|
|
113
120
|
}
|
|
114
121
|
|
|
115
|
-
function
|
|
122
|
+
function r(e, a) {
|
|
116
123
|
if (0 != a.length) {
|
|
117
124
|
let r = [ e ], t = [];
|
|
118
125
|
for (let e = 0; e < a.length - 1; ++e) {
|
|
119
|
-
var i,
|
|
120
|
-
for (i of r) for (var
|
|
121
|
-
t.push(
|
|
126
|
+
var i, s = a[e];
|
|
127
|
+
for (i of r) for (var o of y(i.subdirMap, s)) i.setChildTriggerNodes(o),
|
|
128
|
+
t.push(o);
|
|
122
129
|
if (0 == t.length) return;
|
|
123
130
|
[ r, t ] = [ t, r ], t.length = 0;
|
|
124
131
|
}
|
|
125
|
-
var
|
|
126
|
-
for (
|
|
132
|
+
var n, g = f(a);
|
|
133
|
+
for (n of r) for (var l of y(n.fileMap, g)) n.setChildTriggerNodes(l);
|
|
127
134
|
}
|
|
128
135
|
}
|
|
129
136
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
}
|
|
137
|
+
export {
|
|
138
|
+
h as QyTriggerNode,
|
|
139
|
+
r as updateTriggerNodes
|
|
140
|
+
};
|
package/src/QyUnionNode.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
class
|
|
2
|
-
constructor(
|
|
1
|
+
class i {
|
|
2
|
+
constructor(i = 1 / 0, e = 0) {
|
|
3
3
|
Object.assign(this, {
|
|
4
|
-
limit:
|
|
5
|
-
cursor:
|
|
4
|
+
limit: i,
|
|
5
|
+
cursor: e,
|
|
6
6
|
added: 0,
|
|
7
7
|
count: 0,
|
|
8
8
|
encounteredMap: {}
|
|
@@ -11,95 +11,93 @@ class QueryCounter {
|
|
|
11
11
|
get isFull() {
|
|
12
12
|
return this.limit <= this.added;
|
|
13
13
|
}
|
|
14
|
-
encounterOne(
|
|
14
|
+
encounterOne(i) {
|
|
15
15
|
var {
|
|
16
|
-
encounteredMap:
|
|
17
|
-
isFull:
|
|
16
|
+
encounteredMap: e,
|
|
17
|
+
isFull: t
|
|
18
18
|
} = this;
|
|
19
|
-
if (!
|
|
19
|
+
if (!(t || i in e)) return e[i] = 1, this.countOne();
|
|
20
20
|
}
|
|
21
21
|
countOne() {
|
|
22
22
|
if (this.count++ >= this.cursor) return ++this.added, !0;
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
class
|
|
26
|
+
class e {
|
|
27
27
|
constructor() {
|
|
28
28
|
Object.assign(this, {
|
|
29
29
|
fileCount: 0,
|
|
30
30
|
nodeList: []
|
|
31
31
|
});
|
|
32
32
|
}
|
|
33
|
-
encounterOne(
|
|
34
|
-
if (
|
|
35
|
-
if (r.encounterOne(i)) return e[i] = 1, r.isFull;
|
|
36
|
-
} else e[i] = 1;
|
|
33
|
+
encounterOne(i, e, t) {
|
|
34
|
+
if (!t || t.encounterOne(e)) return i[e] = 1, !0;
|
|
37
35
|
}
|
|
38
36
|
}
|
|
39
37
|
|
|
40
|
-
class
|
|
41
|
-
constructor(
|
|
38
|
+
class o extends e {
|
|
39
|
+
constructor(i) {
|
|
42
40
|
super(), Object.assign(this, {
|
|
43
41
|
dirMap: {},
|
|
44
42
|
checkedDirMap: {}
|
|
45
|
-
}),
|
|
43
|
+
}), i && this.addDir(i);
|
|
46
44
|
}
|
|
47
|
-
containsFile(
|
|
45
|
+
containsFile(e) {
|
|
48
46
|
var {
|
|
49
|
-
fileCount:
|
|
50
|
-
dirMap:
|
|
51
|
-
nodeList:
|
|
47
|
+
fileCount: i,
|
|
48
|
+
dirMap: t,
|
|
49
|
+
nodeList: r
|
|
52
50
|
} = this;
|
|
53
|
-
if (0 <
|
|
54
|
-
for (var n in
|
|
55
|
-
for (let
|
|
51
|
+
if (0 < i) {
|
|
52
|
+
for (var n in t) if (null != t[n].getFile(e)) return !0;
|
|
53
|
+
for (let i = 0; i < r.length; ++i) if (r[i].containsFile(e)) return !0;
|
|
56
54
|
}
|
|
57
55
|
return !1;
|
|
58
56
|
}
|
|
59
|
-
|
|
60
|
-
var
|
|
57
|
+
*getFiles(i, e = {}) {
|
|
58
|
+
var t, r, {
|
|
61
59
|
dirMap: n,
|
|
62
|
-
nodeList:
|
|
63
|
-
} = this
|
|
64
|
-
for (
|
|
65
|
-
var
|
|
66
|
-
|
|
60
|
+
nodeList: s
|
|
61
|
+
} = this;
|
|
62
|
+
for (t in n) {
|
|
63
|
+
var o = n[t];
|
|
64
|
+
if (0 != o.fileCount) for (var a of o.fileNameList) if (!(a in e) && this.encounterOne(e, a, i) && (yield a,
|
|
65
|
+
i?.isFull)) return;
|
|
67
66
|
}
|
|
68
|
-
for (r of
|
|
69
|
-
return s;
|
|
67
|
+
for (r of s) if (yield* r.getFiles(i, e), i?.isFull) return;
|
|
70
68
|
}
|
|
71
|
-
addNode(
|
|
72
|
-
if (
|
|
69
|
+
addNode(i) {
|
|
70
|
+
if (i) {
|
|
73
71
|
var {
|
|
74
|
-
dirMap:
|
|
75
|
-
checkedDirMap:
|
|
76
|
-
nodeList:
|
|
72
|
+
dirMap: e,
|
|
73
|
+
checkedDirMap: t,
|
|
74
|
+
nodeList: r
|
|
77
75
|
} = this;
|
|
78
|
-
if (
|
|
79
|
-
for (var n in
|
|
80
|
-
for (var
|
|
81
|
-
} else
|
|
82
|
-
this.fileCount +=
|
|
76
|
+
if (i instanceof o) {
|
|
77
|
+
for (var n in i.dirMap) e[n] = i.dirMap[n];
|
|
78
|
+
for (var s in r.push(...i.nodeList), i.checkedDirMap) t[s] = 1;
|
|
79
|
+
} else r.push(i);
|
|
80
|
+
this.fileCount += i.fileCount;
|
|
83
81
|
}
|
|
84
82
|
}
|
|
85
|
-
addDir(
|
|
83
|
+
addDir(i) {
|
|
86
84
|
var {
|
|
87
|
-
dirMap:
|
|
88
|
-
checkedDirMap:
|
|
85
|
+
dirMap: e,
|
|
86
|
+
checkedDirMap: t
|
|
89
87
|
} = this, {
|
|
90
|
-
fileCount:
|
|
88
|
+
fileCount: r,
|
|
91
89
|
subdirCount: n,
|
|
92
|
-
fullPathHash:
|
|
93
|
-
} =
|
|
94
|
-
if (!
|
|
95
|
-
if (0 < n) for (var
|
|
96
|
-
0 <
|
|
90
|
+
fullPathHash: s
|
|
91
|
+
} = i;
|
|
92
|
+
if (!(s in e || s in t)) {
|
|
93
|
+
if (0 < n) for (var o of i.subdirList) this.addDir(o);
|
|
94
|
+
0 < r ? (e[s] = i, this.fileCount += r) : t[s] = 1;
|
|
97
95
|
}
|
|
98
96
|
}
|
|
99
97
|
}
|
|
100
98
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
99
|
+
export {
|
|
100
|
+
o as QyUnionNode,
|
|
101
|
+
e as QyQueryNode,
|
|
102
|
+
i as QueryCounter
|
|
103
|
+
};
|