datafolder 1.0.2 → 1.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/README.md +34 -31
  2. package/index.js +1 -3
  3. package/package.json +5 -3
  4. package/sample.js +37 -37
  5. package/scripts/airbnb.js +10 -11
  6. package/scripts/airbnb2.js +10 -11
  7. package/scripts/airbnb2_queries.js +20 -22
  8. package/scripts/airbnb_queries.js +23 -22
  9. package/scripts/downloadFile.js +29 -31
  10. package/scripts/express.js +7 -5
  11. package/scripts/fastify.js +12 -11
  12. package/scripts/flights.js +9 -9
  13. package/scripts/flights2.js +9 -9
  14. package/scripts/flights2_queries.js +107 -105
  15. package/scripts/flights_queries.js +113 -109
  16. package/src/QyAcl.js +153 -152
  17. package/src/QyAclSaver.js +16 -10
  18. package/src/QyAclSaver_Worker.js +21 -15
  19. package/src/QyAclWriter.js +23 -13
  20. package/src/QyBatch.js +298 -263
  21. package/src/QyBinWriter.js +49 -37
  22. package/src/QyCD.js +13 -14
  23. package/src/QyCache.js +177 -140
  24. package/src/QyCompressor.js +71 -45
  25. package/src/QyCompressor_Worker.js +17 -9
  26. package/src/QyDB.js +67 -12
  27. package/src/QyDefaultOptions.js +13 -10
  28. package/src/QyDir.js +89 -53
  29. package/src/QyFetch.js +27 -24
  30. package/src/QyFile.js +89 -77
  31. package/src/QyFileContentUtils.js +129 -132
  32. package/src/QyFileLogger.js +63 -41
  33. package/src/QyIndex.js +175 -197
  34. package/src/QyIndexPropTree.js +25 -25
  35. package/src/QyIntersectNode.js +34 -31
  36. package/src/QyItem.js +54 -52
  37. package/src/QyJsonSaver.js +12 -6
  38. package/src/QyKVData.js +279 -245
  39. package/src/QyKVDataCleaner.js +12 -6
  40. package/src/QyKVDataCleaner_Worker.js +77 -59
  41. package/src/QyKVDataLoader.js +67 -46
  42. package/src/QyKVDataSaver.js +15 -9
  43. package/src/QyKVDataSaver_Worker.js +83 -60
  44. package/src/QyLogFileSaver.js +12 -6
  45. package/src/QyLogFileSaver_Worker.js +24 -14
  46. package/src/QyLogger.js +51 -40
  47. package/src/QyMessageWorker.js +43 -29
  48. package/src/QyMessager.js +74 -54
  49. package/src/QyMessager_Run.js +20 -7
  50. package/src/QyMod.js +15 -11
  51. package/src/QyOpRunner.js +58 -41
  52. package/src/QyQueue.js +35 -30
  53. package/src/QySaver.js +52 -29
  54. package/src/QySnapshotCompactor.js +15 -9
  55. package/src/QySnapshotCompactor_Worker.js +131 -110
  56. package/src/QySnapshotSaver.js +15 -9
  57. package/src/QySnapshotSaver_Worker.js +125 -110
  58. package/src/QySnapshots.js +264 -242
  59. package/src/QyStructure.js +35 -27
  60. package/src/QyTrigger.js +62 -55
  61. package/src/QyUnionNode.js +54 -56
  62. package/src/QyUtils.js +170 -158
  63. package/src/QyWorker.js +34 -20
  64. package/src/hackernews.js +63 -71
  65. package/src/rpc/getFileContent.js +7 -7
  66. package/src/rpc/getSubItems.js +12 -28
  67. package/src/rpc/queryFiles.js +2 -25
  68. package/src/simulate.js +85 -80
  69. package/src/simulate2.js +77 -78
  70. package/src/soc_pokec.js +44 -32
  71. package/static/datafolderui/QyStyledElement.js +12 -12
  72. package/static/datafolderui/QyUIFolderTree.js +20 -20
  73. package/static/datafolderui/QyUIUtils.js +8 -8
  74. package/static/datafolderui/explore-tree.js +31 -31
  75. package/static/datafolderui/folder-tree.js +2 -2
  76. package/static/datafolderui/hscrpt.js +9 -17
  77. package/static/datafolderui/json-view.js +3 -5
  78. package/static/datafolderui/kv-input.js +15 -15
  79. package/static/datafolderui/list-input.js +17 -17
  80. package/static/datafolderui/main-page.js +18 -18
  81. package/static/datafolderui/num-input.js +5 -5
  82. package/static/datafolderui/query-editor.js +6 -1
  83. package/static/datafolderui/querySelectorShadowRoot.js +15 -19
  84. package/static/datafolderui/resizable-input.js +18 -18
@@ -1,117 +1,65 @@
1
- let path = require("node:path"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QySnapshots = require("./QySnapshots.js").QySnapshots, QyKVDataCleaner = require("./QyKVDataCleaner.js").QyKVDataCleaner;
1
+ import {
2
+ join as e
3
+ } from "node:path";
2
4
 
3
- class QySnapshotSaver_Worker extends QyMessageWorker {
5
+ import {
6
+ getDefaultOptions as o
7
+ } from "./QyDefaultOptions.js";
8
+
9
+ import {
10
+ QyMessageWorker as a
11
+ } from "./QyMessageWorker.js";
12
+
13
+ import {
14
+ QySnapshots as n
15
+ } from "./QySnapshots.js";
16
+
17
+ import {
18
+ QyKVDataCleaner as p
19
+ } from "./QyKVDataCleaner.js";
20
+
21
+ class s extends a {
4
22
  constructor({
5
23
  kvFolder: a,
6
24
  options: s
7
25
  }) {
8
26
  super(s = {
9
- ...getDefaultOptions("QySnapshotSaver_Worker"),
27
+ ...o("QySnapshotSaver_Worker"),
10
28
  ...s
11
29
  }, [ "stop", "waitForSnapshotSaved" ]);
12
- var e = path.join(a, "snapshot");
30
+ var t = e(a, "snapshot");
13
31
  Object.assign(this, {
14
32
  kvFolder: a,
15
- snapshotFolder: e,
16
- qySnapshots: new QySnapshots(e, s),
17
- qyKVDataCleaner: new QyKVDataCleaner(this, a, s),
33
+ snapshotFolder: t,
34
+ qySnapshots: new n(t, s),
35
+ qyKVDataCleaner: new p(this, a, s),
18
36
  aclKeyValueMap: {}
19
37
  });
20
38
  }
21
- start(a, s, e) {
22
- this.maxChangeId = a, this.qyKVDataCleaner.start(), s && this.waitForSnapshotSaved(this._createMaxSnapshot(s, a, e));
39
+ start(a, s, t) {
40
+ this.maxChangeId = a, this.qyKVDataCleaner.start(), s && this.waitForSnapshotSaved(h(this, s, a, t));
23
41
  }
24
- castSave(a, s, e) {
42
+ castSave(a, s, t) {
25
43
  var {
26
- aclKeyValueMap: t,
27
- qySnapshots: n
44
+ aclKeyValueMap: e,
45
+ qySnapshots: o
28
46
  } = this;
29
- n.applyCmdObj(s, t, e), this.maxChangeId = a;
47
+ o.applyCmdObj(s, e, t), this.maxChangeId = a;
30
48
  }
31
49
  async saveSnapshot(a) {
32
50
  var {
33
51
  maxChangeId: s,
34
- aclKeyValueMap: e
52
+ aclKeyValueMap: t
35
53
  } = this;
36
- this.aclKeyValueMap = {}, this.waitForSnapshotSaved(this._createMaxSnapshot(a, s, e));
54
+ this.aclKeyValueMap = {}, this.waitForSnapshotSaved(h(this, a, s, t));
37
55
  }
38
56
  releaseSnapshot(a) {
39
- var s, e = this.qySnapshots, {
40
- numToSnapshotMap: t,
41
- keyToSnapshotMap: n
42
- } = e, t = t[a], a = t.keyValueMap;
43
- for (s in delete t.keyValueMap, a) delete n[s];
44
- e.delSnapshot(t), global.gc && global.gc();
45
- }
46
- _createMaxSnapshot(a, s, e) {
47
- var t = this._processOutdates(), a = {
48
- snapshotNum: a,
49
- keyValueMap: e,
50
- outdatedCount: 0,
51
- info: {
52
- maxChangeId: s,
53
- pMVersion: 0,
54
- vLVersion: 0
55
- }
56
- }, e = (this.qySnapshots.addMaxSnapshot(a), this._saveKeyValueMapAndSnapshotInfo(a));
57
- return {
58
- saveMaxSnapshotInfoPromise: e,
59
- outdates: t
60
- };
61
- }
62
- async _saveKeyValueMapAndSnapshotInfo(a) {
63
- var s = this.qySnapshots, {
64
- snapshotNum: a,
65
- info: e
66
- } = (await s.saveKeyValueMap(a), a);
67
- return await s.saveSnapshotInfo(a, e), logger.log("Created snapshot " + a),
68
- {
69
- snapshotNum: a,
70
- maxChangeId: e.maxChangeId
71
- };
72
- }
73
- _processOutdates() {
74
- var a, s = this.qySnapshots, {
75
- emptySnapshotNumList: e,
76
- combiningSnapshotList: t,
77
- pMCompactSnapshotList: n
78
- } = s.calOutdates(), o = {}, p = [], h = [], r = [];
79
- for (a of n) {
80
- var {
81
- snapshotNum: i,
82
- info: l
83
- } = a;
84
- p.push(s.savePosMap(a)), h.push([ i, l.pMVersion ]), o[i] = {
85
- ...l
86
- };
87
- }
88
- n = t.length;
89
- if (0 < n) {
90
- var u, S, y = s.keyToSnapshotMap, {
91
- keyValueMap: M,
92
- info: n,
93
- snapshotNum: d
94
- } = u = s.getMaxCombinedSnapshot(t);
95
- for (S of t) if (u != S) {
96
- var g, {
97
- keyValueMap: v,
98
- snapshotNum: m
99
- } = S;
100
- for (g in v) M[g] = v[g], y[g] = u;
101
- s.delSnapshot(S), e.push(m);
102
- }
103
- p.push(s.saveKeyValueMap(u)), h.push([ d, n.pMVersion ]), r.push(d, n.vLVersion),
104
- o[d] = {
105
- ...n
106
- };
107
- }
108
- return {
109
- emptySnapshotNumList: e,
110
- pMChangeList: h,
111
- vLChange: r,
112
- snapshotInfoMap: o,
113
- promises: p
114
- };
57
+ var s, t = this.qySnapshots, {
58
+ numToSnapshotMap: e,
59
+ keyToSnapshotMap: o
60
+ } = t, e = e[a], a = e.keyValueMap;
61
+ for (s in delete e.keyValueMap, a) delete o[s];
62
+ t.delSnapshot(e), global.gc && global.gc();
115
63
  }
116
64
  async _op_stop() {
117
65
  var {
@@ -124,13 +72,13 @@ class QySnapshotSaver_Worker extends QyMessageWorker {
124
72
  saveMaxSnapshotInfoPromise: a,
125
73
  outdates: s
126
74
  }) {
127
- var e, {
75
+ var t, {
128
76
  snapshotNum: a,
129
- maxChangeId: t
77
+ maxChangeId: e
130
78
  } = await a, {
131
79
  emptySnapshotNumList: s,
132
- pMChangeList: n,
133
- vLChange: o,
80
+ pMChangeList: o,
81
+ vLChange: n,
134
82
  snapshotInfoMap: p,
135
83
  promises: h
136
84
  } = s, {
@@ -138,27 +86,94 @@ class QySnapshotSaver_Worker extends QyMessageWorker {
138
86
  qySnapshots: i,
139
87
  options: l
140
88
  } = this;
141
- for (e in r.cleanAclFiles(a), 0 < h.length && await Promise.all(h), h.length = 0,
142
- p) h.push(i.saveSnapshotInfo(e, p[e]));
89
+ for (t in r.cleanAclFiles(a), 0 < h.length && await Promise.all(h), h.length = 0,
90
+ p) h.push(i.saveSnapshotInfo(t, p[t]));
143
91
  0 < h.length && await Promise.all(h), 0 < s.length && r.removeSnapshotsFiles(s),
144
- 0 < n.length && r.cleanPMFiles(n), 0 < o.length && r.cleanVLFiles(o[0], o[1]);
92
+ 0 < o.length && r.cleanPMFiles(o), 0 < n.length && r.cleanVLFiles(n[0], n[1]);
145
93
  let u;
146
94
  a = i.snapshotCount;
147
- 0 < a && a >= l.maxInMemSnapshotCount && (delete (s = this._getMinSnapshot()).prefixPM,
148
- delete (u = {
95
+ 0 < a && a >= l.maxInMemSnapshotCount && (delete (s = (a => {
96
+ let s = a.qySnapshots.numToSnapshotMap, t;
97
+ for (var e in s) {
98
+ var o = s[e];
99
+ (!t || t.snapshotNum > e) && (t = o);
100
+ }
101
+ return t;
102
+ })(this)).prefixPM, delete (u = {
149
103
  ...s
150
104
  }).keyValueMap, delete s.posMap, s.isOut = !0, logger.info("Sending snapshot " + s.snapshotNum)),
151
- this.castParent("onSnapshotSavedChangeId", t, u);
105
+ this.castParent("onSnapshotSavedChangeId", e, u);
152
106
  }
153
- _getMinSnapshot() {
154
- var a, s = this.qySnapshots.numToSnapshotMap;
155
- let e;
156
- for (a in s) {
157
- var t = s[a];
158
- (!e || e.snapshotNum > a) && (e = t);
107
+ }
108
+
109
+ function h(a, s, t, e) {
110
+ var o = (a => {
111
+ var s, t = a.qySnapshots, {
112
+ emptySnapshotNumList: e,
113
+ combiningSnapshotList: a,
114
+ pMCompactSnapshotList: o
115
+ } = t.calOutdates(), n = {}, p = [], h = [], r = [];
116
+ for (s of o) {
117
+ var {
118
+ snapshotNum: i,
119
+ info: l
120
+ } = s;
121
+ p.push(t.savePosMap(s)), h.push([ i, l.pMVersion ]), n[i] = {
122
+ ...l
123
+ };
159
124
  }
160
- return e;
161
- }
125
+ let u = a.length, m;
126
+ if (0 < u) {
127
+ var S, y = t.keyToSnapshotMap, {
128
+ keyValueMap: d,
129
+ info: o,
130
+ snapshotNum: v
131
+ } = m = t.getMaxCombinedSnapshot(a);
132
+ for (S of a) if (m != S) {
133
+ var g, {
134
+ keyValueMap: M,
135
+ snapshotNum: f
136
+ } = S;
137
+ for (g in M) d[g] = M[g], y[g] = m;
138
+ t.delSnapshot(S), e.push(f);
139
+ }
140
+ p.push(t.saveKeyValueMap(m)), h.push([ v, o.pMVersion ]), r.push(v, o.vLVersion),
141
+ n[v] = {
142
+ ...o
143
+ };
144
+ }
145
+ return {
146
+ emptySnapshotNumList: e,
147
+ pMChangeList: h,
148
+ vLChange: r,
149
+ snapshotInfoMap: n,
150
+ promises: p
151
+ };
152
+ })(a), s = {
153
+ snapshotNum: s,
154
+ keyValueMap: e,
155
+ outdatedCount: 0,
156
+ info: {
157
+ maxChangeId: t,
158
+ pMVersion: 0,
159
+ vLVersion: 0
160
+ }
161
+ }, e = (a.qySnapshots.addMaxSnapshot(s), (async (a, s) => {
162
+ await (a = a.qySnapshots).saveKeyValueMap(s);
163
+ var {
164
+ snapshotNum: s,
165
+ info: t
166
+ } = s;
167
+ return await a.saveSnapshotInfo(s, t), logger.log("Created snapshot " + s),
168
+ {
169
+ snapshotNum: s,
170
+ maxChangeId: t.maxChangeId
171
+ };
172
+ })(a, s));
173
+ return {
174
+ saveMaxSnapshotInfoPromise: e,
175
+ outdates: o
176
+ };
162
177
  }
163
178
 
164
- module.exports = QySnapshotSaver_Worker;
179
+ export default s;