datafolder 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +169 -0
  2. package/index.js +3 -0
  3. package/package.json +27 -0
  4. package/sample.js +37 -0
  5. package/scripts/airbnb.js +73 -0
  6. package/scripts/airbnb2.js +74 -0
  7. package/scripts/airbnb2_queries.js +231 -0
  8. package/scripts/airbnb_queries.js +264 -0
  9. package/scripts/downloadFile.js +42 -0
  10. package/scripts/express.js +46 -0
  11. package/scripts/fastify.js +79 -0
  12. package/scripts/flights.js +75 -0
  13. package/scripts/flights2.js +81 -0
  14. package/scripts/flights2_queries.js +453 -0
  15. package/scripts/flights_queries.js +403 -0
  16. package/src/QyAcl.js +252 -0
  17. package/src/QyAclSaver.js +14 -0
  18. package/src/QyAclSaver_Worker.js +29 -0
  19. package/src/QyAclWriter.js +19 -0
  20. package/src/QyBatch.js +501 -0
  21. package/src/QyBinWriter.js +140 -0
  22. package/src/QyCD.js +61 -0
  23. package/src/QyCache.js +212 -0
  24. package/src/QyCompressor.js +60 -0
  25. package/src/QyCompressor_Worker.js +13 -0
  26. package/src/QyDB.js +122 -0
  27. package/src/QyDefaultOptions.js +59 -0
  28. package/src/QyDir.js +192 -0
  29. package/src/QyFetch.js +38 -0
  30. package/src/QyFile.js +124 -0
  31. package/src/QyFileContentUtils.js +175 -0
  32. package/src/QyFileLogger.js +58 -0
  33. package/src/QyIndex.js +237 -0
  34. package/src/QyIndexPropTree.js +92 -0
  35. package/src/QyIntersectNode.js +48 -0
  36. package/src/QyItem.js +105 -0
  37. package/src/QyJsonSaver.js +20 -0
  38. package/src/QyKVData.js +289 -0
  39. package/src/QyKVDataCleaner.js +17 -0
  40. package/src/QyKVDataCleaner_Worker.js +94 -0
  41. package/src/QyKVDataLoader.js +65 -0
  42. package/src/QyKVDataSaver.js +17 -0
  43. package/src/QyKVDataSaver_Worker.js +81 -0
  44. package/src/QyLogFileSaver.js +14 -0
  45. package/src/QyLogFileSaver_Worker.js +26 -0
  46. package/src/QyLogger.js +76 -0
  47. package/src/QyMessageWorker.js +41 -0
  48. package/src/QyMessager.js +84 -0
  49. package/src/QyMessager_Run.js +13 -0
  50. package/src/QyMod.js +15 -0
  51. package/src/QyOpRunner.js +73 -0
  52. package/src/QyQueue.js +60 -0
  53. package/src/QySaver.js +83 -0
  54. package/src/QySnapshotCompactor.js +17 -0
  55. package/src/QySnapshotCompactor_Worker.js +151 -0
  56. package/src/QySnapshotSaver.js +17 -0
  57. package/src/QySnapshotSaver_Worker.js +164 -0
  58. package/src/QySnapshots.js +354 -0
  59. package/src/QyStructure.js +50 -0
  60. package/src/QyTrigger.js +133 -0
  61. package/src/QyUnionNode.js +105 -0
  62. package/src/QyUtils.js +250 -0
  63. package/src/QyWorker.js +33 -0
  64. package/src/hackernews.js +93 -0
  65. package/src/rpc/evalCode.js +9 -0
  66. package/src/rpc/getFileContent.js +16 -0
  67. package/src/rpc/getSubItems.js +32 -0
  68. package/src/rpc/queryFiles.js +33 -0
  69. package/src/rpc/reloadRpc.js +1 -0
  70. package/src/simulate.js +114 -0
  71. package/src/simulate2.js +110 -0
  72. package/src/soc_pokec.js +43 -0
  73. package/static/bootstrap-5.3.8-dist/css/bootstrap.min.css +6 -0
  74. package/static/bootstrap-5.3.8-dist/js/bootstrap.bundle.js +6312 -0
  75. package/static/datafolderui/QyStyledElement.js +30 -0
  76. package/static/datafolderui/QyUIFolderTree.js +203 -0
  77. package/static/datafolderui/QyUIUtils.js +42 -0
  78. package/static/datafolderui/display-tree.js +108 -0
  79. package/static/datafolderui/explore-tree.js +122 -0
  80. package/static/datafolderui/folder-tree.js +5 -0
  81. package/static/datafolderui/hscrpt.js +25 -0
  82. package/static/datafolderui/index.css +6 -0
  83. package/static/datafolderui/index.html +20 -0
  84. package/static/datafolderui/json-view.js +74 -0
  85. package/static/datafolderui/kv-input.js +66 -0
  86. package/static/datafolderui/list-input.js +52 -0
  87. package/static/datafolderui/main-page.js +98 -0
  88. package/static/datafolderui/num-input.js +27 -0
  89. package/static/datafolderui/query-editor.js +229 -0
  90. package/static/datafolderui/querySelectorShadowRoot.js +24 -0
  91. package/static/datafolderui/resizable-input.js +67 -0
  92. package/static/favicon.ico +0 -0
  93. package/static/icons-1.13.1/font/bootstrap-icons.min.css +5 -0
  94. package/static/icons-1.13.1/font/fonts/bootstrap-icons.woff +0 -0
  95. package/static/icons-1.13.1/font/fonts/bootstrap-icons.woff2 +0 -0
  96. package/static/json-view/dist/jsonview.js +433 -0
  97. package/static/tree.js/tree.css +106 -0
  98. package/static/tree.js/tree.js +374 -0
  99. package/static/w2ui/dist/w2ui.min.css +2 -0
  100. package/static/w2ui/src/less/icons/icons.json +1 -0
  101. package/static/w2ui/src/less/icons/preview.html +33 -0
  102. package/static/w2ui/src/less/icons/readme.md +15 -0
  103. package/static/w2ui/src/less/icons/svg/box.svg +3 -0
  104. package/static/w2ui/src/less/icons/svg/check.svg +3 -0
  105. package/static/w2ui/src/less/icons/svg/colors.svg +3 -0
  106. package/static/w2ui/src/less/icons/svg/columns.svg +5 -0
  107. package/static/w2ui/src/less/icons/svg/cross.svg +4 -0
  108. package/static/w2ui/src/less/icons/svg/drop.svg +6 -0
  109. package/static/w2ui/src/less/icons/svg/empty.svg +3 -0
  110. package/static/w2ui/src/less/icons/svg/eye-dropper.svg +4 -0
  111. package/static/w2ui/src/less/icons/svg/info.svg +8 -0
  112. package/static/w2ui/src/less/icons/svg/paste.svg +4 -0
  113. package/static/w2ui/src/less/icons/svg/pencil.svg +5 -0
  114. package/static/w2ui/src/less/icons/svg/plus.svg +3 -0
  115. package/static/w2ui/src/less/icons/svg/reload.svg +5 -0
  116. package/static/w2ui/src/less/icons/svg/search.svg +3 -0
  117. package/static/w2ui/src/less/icons/svg/settings.svg +8 -0
  118. package/static/w2ui/src/less/icons/w2ui-font.css +36 -0
  119. package/static/w2ui/src/less/icons/w2ui-font.woff +0 -0
  120. package/static/w2ui/src/less/src/buttons.less +184 -0
  121. package/static/w2ui/src/less/src/common.less +541 -0
  122. package/static/w2ui/src/less/src/fields.less +302 -0
  123. package/static/w2ui/src/less/src/form.less +319 -0
  124. package/static/w2ui/src/less/src/grid.less +1057 -0
  125. package/static/w2ui/src/less/src/icons.less +37 -0
  126. package/static/w2ui/src/less/src/layout.less +86 -0
  127. package/static/w2ui/src/less/src/mixins.less +87 -0
  128. package/static/w2ui/src/less/src/popup.less +277 -0
  129. package/static/w2ui/src/less/src/sidebar.less +343 -0
  130. package/static/w2ui/src/less/src/tabs.less +145 -0
  131. package/static/w2ui/src/less/src/toolbar.less +219 -0
  132. package/static/w2ui/src/less/src/tooltip.less +948 -0
  133. package/static/w2ui/src/less/w2ui.less +196 -0
  134. package/static/w2ui/src/locale/az-az.json +114 -0
  135. package/static/w2ui/src/locale/ba-ba.json +114 -0
  136. package/static/w2ui/src/locale/bg-bg.json +114 -0
  137. package/static/w2ui/src/locale/ca-es.json +114 -0
  138. package/static/w2ui/src/locale/da-dk.json +114 -0
  139. package/static/w2ui/src/locale/de-de.json +114 -0
  140. package/static/w2ui/src/locale/en-gb.json +17 -0
  141. package/static/w2ui/src/locale/en-us.json +17 -0
  142. package/static/w2ui/src/locale/es-ar.json +115 -0
  143. package/static/w2ui/src/locale/es-cr.json +119 -0
  144. package/static/w2ui/src/locale/es-es.json +114 -0
  145. package/static/w2ui/src/locale/es-mx.json +114 -0
  146. package/static/w2ui/src/locale/fr-fr.json +114 -0
  147. package/static/w2ui/src/locale/gl-es.json +114 -0
  148. package/static/w2ui/src/locale/hr-hr.json +114 -0
  149. package/static/w2ui/src/locale/hu-hu.json +114 -0
  150. package/static/w2ui/src/locale/id-id.json +114 -0
  151. package/static/w2ui/src/locale/it-it.json +114 -0
  152. package/static/w2ui/src/locale/ja-jp.json +114 -0
  153. package/static/w2ui/src/locale/kk-kz.json +116 -0
  154. package/static/w2ui/src/locale/ko-kr.json +114 -0
  155. package/static/w2ui/src/locale/lt-lt.json +114 -0
  156. package/static/w2ui/src/locale/nl-nl.json +114 -0
  157. package/static/w2ui/src/locale/no-no.json +114 -0
  158. package/static/w2ui/src/locale/pl-pl.json +114 -0
  159. package/static/w2ui/src/locale/pt-br.json +114 -0
  160. package/static/w2ui/src/locale/ru-ru.json +116 -0
  161. package/static/w2ui/src/locale/sk-sk.json +114 -0
  162. package/static/w2ui/src/locale/sl-si.json +116 -0
  163. package/static/w2ui/src/locale/sr-rs.json +114 -0
  164. package/static/w2ui/src/locale/tr-tr.json +114 -0
  165. package/static/w2ui/src/locale/uk-ua.json +114 -0
  166. package/static/w2ui/src/locale/zh-cn.json +114 -0
  167. package/static/w2ui/src/locale/zh-tw.json +114 -0
  168. package/static/w2ui/src/query.js +583 -0
  169. package/static/w2ui/src/readme.md +11 -0
  170. package/static/w2ui/src/w2base.js +311 -0
  171. package/static/w2ui/src/w2compat.js +207 -0
  172. package/static/w2ui/src/w2field.js +1854 -0
  173. package/static/w2ui/src/w2form.js +2411 -0
  174. package/static/w2ui/src/w2grid.js +8916 -0
  175. package/static/w2ui/src/w2layout.js +1194 -0
  176. package/static/w2ui/src/w2locale.js +129 -0
  177. package/static/w2ui/src/w2popup.js +909 -0
  178. package/static/w2ui/src/w2sidebar.js +1769 -0
  179. package/static/w2ui/src/w2tabs.js +669 -0
  180. package/static/w2ui/src/w2toolbar.js +1180 -0
  181. package/static/w2ui/src/w2tooltip.js +3134 -0
  182. package/static/w2ui/src/w2utils.js +2457 -0
@@ -0,0 +1,289 @@
1
+ let path = require("node:path"), fsPromises = require("node:fs/promises"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyKVDataSaver = require("./QyKVDataSaver.js").QyKVDataSaver, QySnapshotCompactor = require("./QySnapshotCompactor.js").QySnapshotCompactor, QyKVDataCleaner = require("./QyKVDataCleaner.js").QyKVDataCleaner, {
2
+ arrayLast,
3
+ readBufferAsync,
4
+ sleep
5
+ } = require("./QyUtils.js"), Idle = 1, WaitForCompactUpdates = 2, WaitForSnapshotSaved = 3;
6
+
7
+ class QyKVData {
8
+ constructor(a, t, s) {
9
+ t = {
10
+ ...getDefaultOptions("QyKVData"),
11
+ ...t
12
+ };
13
+ var o = path.join(a, "snapshot");
14
+ Object.assign(this, {
15
+ qyCache: s,
16
+ options: t,
17
+ kvFolder: a,
18
+ snapshotFolder: o,
19
+ qyKVDataSaver: new QyKVDataSaver(this, a, t),
20
+ qySnapshotCompactor: new QySnapshotCompactor(this, a, t),
21
+ qyKVDataCleaner: new QyKVDataCleaner(this, a, t),
22
+ compactState: Idle,
23
+ compactMaxChangeId: 0,
24
+ snapshotSavedChangeId: 0,
25
+ snapshotInfoNumList: [],
26
+ emptySnapshotNumList: []
27
+ });
28
+ }
29
+ start(a, t, s, o) {
30
+ var {
31
+ options: e,
32
+ qyKVDataSaver: n,
33
+ qySnapshotCompactor: p,
34
+ qyKVDataCleaner: i
35
+ } = this, n = (n.start(a, s.maxSnapshotNum, t, o), p.start(), i.start(),
36
+ setInterval(() => this._compactSnapshots(), e.compactSnapshotsInterval)), t = setInterval(() => this._unloadMemory(), e.unloadMemInterval);
37
+ return Object.assign(this, {
38
+ maxChangeId: a,
39
+ qySnapshots: s,
40
+ aclKeyValueMap: o,
41
+ compactInterval: n,
42
+ unloadInterval: t
43
+ }), e.bgLoadPosMaps && this._bgLoadPosMaps(), this;
44
+ }
45
+ async stop() {
46
+ this.isStopping = !0;
47
+ var {
48
+ qyKVDataSaver: a,
49
+ qySnapshotCompactor: t,
50
+ qySnapshots: s,
51
+ compactInterval: o,
52
+ unloadInterval: e,
53
+ qyKVDataCleaner: n
54
+ } = this;
55
+ clearInterval(o), clearInterval(e), await a.stop(), s && s.closeAllFDs(),
56
+ await t.stop(), await n.stop();
57
+ }
58
+ save(a, t, s, o) {
59
+ var e = this.qyKVDataSaver;
60
+ if (o) return e.callSave(a, t, s);
61
+ e.castSave(a, t, s);
62
+ }
63
+ increaseChangeId() {
64
+ return ++this.maxChangeId;
65
+ }
66
+ decreaseChangeId() {
67
+ --this.maxChangeId;
68
+ }
69
+ getValueSync(a) {
70
+ var {
71
+ qySnapshots: t,
72
+ aclKeyValueMap: s
73
+ } = this;
74
+ return a in s ? s[a] : t.getValueSync(a);
75
+ }
76
+ removeKey(a) {
77
+ var {
78
+ qySnapshots: t,
79
+ aclKeyValueMap: s
80
+ } = this;
81
+ if (!(a in s)) return t.removeKey(a);
82
+ s[a] = void 0;
83
+ }
84
+ onSnapshotSavedChangeId(a, t) {
85
+ this.snapshotSavedChangeId = a, this.compactState == WaitForSnapshotSaved && a >= this.compactMaxChangeId && this._finishCompactUpdates(),
86
+ t && (this._addPMOnlySnapshot(t), this.qyKVDataSaver.releaseSnapshot(t.snapshotNum));
87
+ }
88
+ onCompactUpdates(a) {
89
+ this.compactState == WaitForCompactUpdates && (this._processCompactUpdates(a),
90
+ this.snapshotSavedChangeId >= this.compactMaxChangeId ? this._finishCompactUpdates() : this.compactState = WaitForSnapshotSaved);
91
+ }
92
+ _addPMOnlySnapshot(a) {
93
+ var t, s = this.qySnapshots, o = s.keyToSnapshotMap;
94
+ for (t in a.posMap) {
95
+ var e = o[t];
96
+ e && s.removeKeyFromSnapshot(e, t);
97
+ }
98
+ s.addMaxSnapshot(a), logger.info("addPMOnlySnapshot: " + s.snapshotMaxChangeId);
99
+ }
100
+ _finishCompactUpdates() {
101
+ var {
102
+ qySnapshotCompactor: a,
103
+ qyKVDataCleaner: t,
104
+ snapshotInfoNumList: s,
105
+ emptySnapshotNumList: o
106
+ } = this;
107
+ a.saveSnapshotInfos(s), (s.length = 0) < o.length && (t.removeSnapshotsFiles(o),
108
+ o.length = 0), this.compactState = Idle;
109
+ }
110
+ _processCompactUpdates(a) {
111
+ var {
112
+ pMUpdates: a,
113
+ vLUpdates: t,
114
+ combineUpdate: s
115
+ } = a;
116
+ a && this._processPMUpdates(a), t && this._processVLUpdates(t), s && this._processCombineUpdate(s);
117
+ }
118
+ _processCombineUpdate(a) {
119
+ var {
120
+ qySnapshots: o,
121
+ snapshotInfoNumList: t,
122
+ emptySnapshotNumList: s
123
+ } = this, {
124
+ numToSnapshotMap: e,
125
+ keyToSnapshotMap: n
126
+ } = o;
127
+ let {
128
+ snapshotNum: p,
129
+ combinedSnapshotNumList: i,
130
+ posMap: r
131
+ } = a;
132
+ var h, c = e[p];
133
+ let {
134
+ posMap: l,
135
+ info: d
136
+ } = c;
137
+ for (h in l) l[h] = r[h];
138
+ this._countLVCompact(c);
139
+ let u = d.totalCount, m = c.outdatedCount;
140
+ for (let s of i) {
141
+ var S, y = e[s];
142
+ let {
143
+ posMap: a,
144
+ info: t
145
+ } = y;
146
+ for (S in a) l[S] = r[S], n[S] = c;
147
+ this._countLVCompact(y), u += t.totalCount, m += y.outdatedCount, o.delSnapshot(y);
148
+ }
149
+ s.push(...i), (u == m ? s : (++d.pMVersion, ++d.vLVersion, d.totalCount = u,
150
+ c.outdatedCount = m, t)).push(p);
151
+ }
152
+ _processVLUpdates(a) {
153
+ var t, {
154
+ qySnapshots: s,
155
+ snapshotInfoNumList: o,
156
+ emptySnapshotNumList: e
157
+ } = this, n = s.numToSnapshotMap;
158
+ for (t in a) {
159
+ var p = n[t];
160
+ if (s.isEmptySnapshot(p)) s.delSnapshot(p), e.push(t); else {
161
+ var i, r, {
162
+ posMap: h,
163
+ prefixPM: c
164
+ } = a[t], {
165
+ posMap: l,
166
+ prefixPM: d,
167
+ info: u
168
+ } = p;
169
+ for (i in l) l[i] = h[i];
170
+ for (r in d) d[r] = c[r];
171
+ ++u.pMVersion, ++u.vLVersion, this._countLVCompact(p), o.push(t);
172
+ }
173
+ }
174
+ }
175
+ _countLVCompact(a) {
176
+ var {
177
+ info: t,
178
+ compactOutedCount: s
179
+ } = a;
180
+ t.totalCount -= t.outdatedCount + s, t.outdatedCount = 0, a.outdatedCount -= s,
181
+ a.compactOutedCount = 0;
182
+ }
183
+ _countPMCompact(a) {
184
+ var {
185
+ info: t,
186
+ compactOutedCount: s
187
+ } = a;
188
+ t.outdatedCount += s, a.outdatedCount -= s, a.compactOutedCount = 0;
189
+ }
190
+ _processPMUpdates(a) {
191
+ var t, {
192
+ qySnapshots: s,
193
+ snapshotInfoNumList: o,
194
+ emptySnapshotNumList: e
195
+ } = this, n = s.numToSnapshotMap;
196
+ for (t in a) {
197
+ var p = n[t];
198
+ if (s.isEmptySnapshot(p)) s.delSnapshot(p), e.push(t); else {
199
+ var i, r = a[t], {
200
+ prefixPM: h,
201
+ info: c
202
+ } = p;
203
+ for (i in h) h[i] = r[i];
204
+ ++c.pMVersion, this._countPMCompact(p), o.push(t);
205
+ }
206
+ }
207
+ }
208
+ _compactSnapshots() {
209
+ if (this.compactState == Idle) {
210
+ var {
211
+ maxChangeId: a,
212
+ qySnapshots: t,
213
+ qySnapshotCompactor: s
214
+ } = this, {
215
+ emptySnapshotNumList: t,
216
+ pMCompactSnapshotList: o,
217
+ vLCompactSnapshotList: e,
218
+ combiningSnapshotList: n
219
+ } = t.calOutdates();
220
+ if (0 < t.length && this.emptySnapshotNumList.push(...t), 0 < n.length || 0 < o.length || 0 < e.length) {
221
+ Object.assign(this, {
222
+ compactState: WaitForCompactUpdates,
223
+ compactMaxChangeId: a
224
+ });
225
+ for (var p of [ o, e, n ]) for (var i of p) i.compactOutedCount = i.outdatedCount;
226
+ s.compactSnapshots(o, e, n);
227
+ }
228
+ }
229
+ }
230
+ async _bgLoadPosMaps() {
231
+ for (var a of Object.keys(this.qySnapshots.numToSnapshotMap).sort((a, t) => t - a)) {
232
+ if (await this._bgLoadPosMap(a), this.isStopping) break;
233
+ if (await sleep(0), this.isStopping) break;
234
+ }
235
+ }
236
+ async _bgLoadPosMap(a, t, s, o, e) {
237
+ if (this.isStopping) e && e.close(); else {
238
+ var n = this.qySnapshots, p = n.numToSnapshotMap, i = p[a];
239
+ if (i) {
240
+ var {
241
+ info: r,
242
+ prefixPM: h
243
+ } = i;
244
+ if (0 == (t = t || Object.keys(h).sort().reverse()).length) e && e.close(); else {
245
+ if (null == s || s != r.pMVersion) return e && e.close(), o = n.getPMFilePath(i),
246
+ this._bgLoadPosMap(a, t, r.pMVersion, o, await fsPromises.open(o));
247
+ for (;0 < t.length; ) {
248
+ var c = arrayLast(t), l = h[c];
249
+ if (l) {
250
+ var [ l, d ] = l, l = await readBufferAsync(o, e, l, d);
251
+ if (this.isStopping || !p[a]) return void e.close();
252
+ if (s != r.pMVersion) return e.close(), this._bgLoadPosMap(a, t);
253
+ c in h && n.loadPartialPosMapBuffer(i, c, l);
254
+ }
255
+ t.pop();
256
+ }
257
+ e.close(), logger.info("bgLoadPosMap:" + a);
258
+ }
259
+ } else e && e.close();
260
+ }
261
+ }
262
+ async _unloadMemory() {
263
+ if (!this.unloadingMemory) {
264
+ this.unloadingMemory = !0;
265
+ var {
266
+ qyCache: s,
267
+ options: o
268
+ } = this;
269
+ let a = 0, t = 0;
270
+ if (s) {
271
+ var e, n = o.maxFileCountToUnload, o = s.rootDir;
272
+ t = this._unloadDirFilesMem(o), a = t;
273
+ for (e of o.allLoadedSubdirs()) if ((t += this._unloadDirFilesMem(e)) > n && (a += t,
274
+ t = 0, await sleep(0), this.isStopping)) break;
275
+ }
276
+ a += t, global.gc && global.gc(), this.unloadingMemory = !1;
277
+ }
278
+ }
279
+ _unloadDirFilesMem(a) {
280
+ var t = this.qySnapshots.snapshotMaxChangeId;
281
+ let s = 0;
282
+ if (a.fileMapLoaded) for (var o of a.fileList) ++s, o.cChangeId <= t && (o.visited ? o.visited = !1 : o.unloadFileContent());
283
+ return s;
284
+ }
285
+ }
286
+
287
+ Object.assign(module.exports, {
288
+ QyKVData: QyKVData
289
+ });
@@ -0,0 +1,17 @@
1
+ let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
2
+
3
+ class QyKVDataCleaner extends QyMessager {
4
+ constructor(e, s, a) {
5
+ super(e, "QyKVDataCleaner_Worker.js", {
6
+ kvFolder: s,
7
+ options: a = {
8
+ ...getDefaultOptions("QyKVDataCleaner"),
9
+ ...a
10
+ }
11
+ }, a, [], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
12
+ }
13
+ }
14
+
15
+ Object.assign(module.exports, {
16
+ QyKVDataCleaner: QyKVDataCleaner
17
+ });
@@ -0,0 +1,94 @@
1
+ let path = require("node:path"), fsPromises = require("node:fs/promises"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, {
2
+ folderOrFileExists,
3
+ ensureDir
4
+ } = require("./QyUtils.js"), qyCompressor = require("./QyCompressor.js");
5
+
6
+ class QyKVDataCleaner_Worker extends QyMessageWorker {
7
+ constructor({
8
+ kvFolder: e,
9
+ options: s
10
+ }) {
11
+ super(s = {
12
+ ...getDefaultOptions("QyKVDataCleaner_Worker"),
13
+ ...s
14
+ }, [ "start", "stop" ], [ "cleanAclFiles", "cleanPMFiles", "cleanVLFiles", "removeSnapshotsFiles" ]);
15
+ var s = path.join(e, "history"), a = path.join(e, "snapshot");
16
+ Object.assign(this, {
17
+ kvFolder: e,
18
+ snapshotFolder: a,
19
+ snapshotInfoFolder: path.join(a, "info"),
20
+ aclFolder: path.join(e, "acl"),
21
+ historyAclFolder: path.join(s, "acl"),
22
+ historySnapshotFolder: path.join(s, "snapshot")
23
+ });
24
+ }
25
+ _op_start() {}
26
+ _op_stop() {}
27
+ async _op_cleanAclFiles(e) {
28
+ await this._cleanOutdatedAclFiles(e);
29
+ }
30
+ async _op_cleanPMFiles(e) {
31
+ await Promise.allSettled(e.map(([ e, s ]) => this._cleanOutdatedPMFiles(e, s)));
32
+ }
33
+ async _op_cleanVLFiles(e, s) {
34
+ await this._cleanOutdatedVLFiles(e, s);
35
+ }
36
+ async _op_removeSnapshotsFiles(e) {
37
+ let {
38
+ snapshotFolder: s,
39
+ snapshotInfoFolder: a
40
+ } = this;
41
+ await Promise.all(e.map(async e => {
42
+ await fsPromises.rm(path.join(a, e + ".json")), await fsPromises.rm(path.join(s, "" + e), {
43
+ recursive: !0
44
+ });
45
+ }));
46
+ }
47
+ async _cleanOutdatedAclFiles(e) {
48
+ var s = this.aclFolder;
49
+ if (await folderOrFileExists(s)) {
50
+ var a, t = [];
51
+ for (a of (await fsPromises.readdir(s)).filter(e => /^[1-9]/.test(e))) parseInt(a) <= e && t.push(this._cleanOutdatedAclFile(a));
52
+ 0 < t.length && await Promise.allSettled(t);
53
+ }
54
+ }
55
+ _cleanOutdatedPMFiles(s, a) {
56
+ return Promise.all([ "pM", "prefixPM" ].map(e => this._cleanOutdatedSnapshotFiles(s, a, e)));
57
+ }
58
+ _cleanOutdatedVLFiles(e, s) {
59
+ return this._cleanOutdatedSnapshotFiles(e, s, "vL");
60
+ }
61
+ async _cleanOutdatedSnapshotFiles(s, a, t) {
62
+ var {
63
+ snapshotFolder: e,
64
+ historySnapshotFolder: r
65
+ } = this, i = path.join(e, "" + s);
66
+ if (await folderOrFileExists(i)) {
67
+ let e;
68
+ var o, l = [];
69
+ for (o of await fsPromises.readdir(i)) {
70
+ var n = new RegExp("^([0-9]+)_" + t).exec(o);
71
+ n && (n = parseInt(n[1])) < a && (1 == n ? (e || (e = path.join(r, "" + s),
72
+ await ensureDir(e)), l.push(this._cleanOutdatedFile(o, i, e))) : (n = path.join(i, o),
73
+ logger.info("Removing " + n), l.push(fsPromises.rm(n))));
74
+ }
75
+ 0 < l.length && await Promise.allSettled(l);
76
+ }
77
+ }
78
+ async _cleanOutdatedAclFile(e) {
79
+ var {
80
+ aclFolder: s,
81
+ historyAclFolder: a
82
+ } = this;
83
+ return await ensureDir(a), this._cleanOutdatedFile(e, s, a);
84
+ }
85
+ _cleanOutdatedFile(e, s, a) {
86
+ s = path.join(s, e), a = path.join(a, e);
87
+ return this.options.compressRedundantFile ? qyCompressor.compressFile(s, a + qyCompressor.Zip_File_Extension, {
88
+ override: !0,
89
+ removeSrcAfterSuccess: !0
90
+ }) : (logger.info(`rename ${s} to ` + a), fsPromises.rename(s, a));
91
+ }
92
+ }
93
+
94
+ module.exports = QyKVDataCleaner_Worker;
@@ -0,0 +1,65 @@
1
+ let fsPromises = require("node:fs/promises"), path = require("node:path"), folderOrFileExists = require("./QyUtils.js").folderOrFileExists, readAclBuffer = require("./QyAcl.js").readAclBuffer, QySnapshots = require("./QySnapshots.js").QySnapshots, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
2
+
3
+ class QyKVDataLoader {
4
+ constructor(a, e) {
5
+ e = {
6
+ ...getDefaultOptions("QyKVDataLoader"),
7
+ ...e
8
+ };
9
+ var s = path.join(a, "snapshot");
10
+ Object.assign(this, {
11
+ kvFolder: a,
12
+ options: e,
13
+ aclFolder: path.join(a, "acl"),
14
+ snapshotFolder: s,
15
+ loadedMaxAclNum: 0,
16
+ aclKeyValueMap: {},
17
+ qySnapshots: new QySnapshots(s, e)
18
+ });
19
+ }
20
+ async load() {
21
+ var a = this.qySnapshots, [ e ] = await Promise.all([ this._listAclFiles(), a.loadSnapshotInfos() ]);
22
+ let {
23
+ maxSnapshotNum: s,
24
+ snapshotMaxChangeId: t
25
+ } = a;
26
+ this.maxChangeId = t, e && await this._loadAclFiles(e.filter(a => a > s));
27
+ var {
28
+ maxChangeId: e,
29
+ loadedMaxAclNum: l,
30
+ aclKeyValueMap: i
31
+ } = this;
32
+ return {
33
+ maxChangeId: e,
34
+ loadedMaxAclNum: l,
35
+ qySnapshots: a,
36
+ aclKeyValueMap: i
37
+ };
38
+ }
39
+ async _loadAclFiles(e) {
40
+ e.sort((a, e) => a - e);
41
+ var {
42
+ aclKeyValueMap: s,
43
+ qySnapshots: t,
44
+ options: a
45
+ } = this, l = a.autoRepairAclFile, i = await Promise.all(e.map(async a => fsPromises.readFile(this._getAclFilePath(a))));
46
+ for (let a = 0; a < e.length; ++a) {
47
+ var r = e[a], o = this._getAclFilePath(r), o = await readAclBuffer(o, i[a], l, this.maxChangeId);
48
+ if (0 < o.length) {
49
+ for (var [ n, h ] of o) t.applyCmdObj(h, s), this.maxChangeId = n;
50
+ this.loadedMaxAclNum = r;
51
+ }
52
+ }
53
+ }
54
+ async _listAclFiles() {
55
+ var a = this.aclFolder;
56
+ if (await folderOrFileExists(a)) return (await fsPromises.readdir(a)).filter(a => /^[1-9]/.test(a)).map(a => parseInt(a));
57
+ }
58
+ _getAclFilePath(a) {
59
+ return path.join(this.aclFolder, a + "_acl.txt");
60
+ }
61
+ }
62
+
63
+ Object.assign(module.exports, {
64
+ QyKVDataLoader: QyKVDataLoader
65
+ });
@@ -0,0 +1,17 @@
1
+ let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
2
+
3
+ class QyKVDataSaver extends QyMessager {
4
+ constructor(e, a, s) {
5
+ super(e, "QyKVDataSaver_Worker.js", {
6
+ kvFolder: a,
7
+ options: s = {
8
+ ...getDefaultOptions("QyKVDataSaver"),
9
+ ...s
10
+ }
11
+ }, s, [ "callSave" ], [ "castSave", "releaseSnapshot" ]);
12
+ }
13
+ }
14
+
15
+ Object.assign(module.exports, {
16
+ QyKVDataSaver: QyKVDataSaver
17
+ });
@@ -0,0 +1,81 @@
1
+ let path = require("node:path"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyAclSaver = require("./QyAclSaver.js").QyAclSaver, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QySnapshotSaver = require("./QySnapshotSaver.js").QySnapshotSaver, delCmdKey = require("./QyAcl.js").delCmdKey;
2
+
3
+ class QyKVDataSaver_Worker extends QyMessageWorker {
4
+ constructor({
5
+ kvFolder: e,
6
+ options: a
7
+ }) {
8
+ super(a = {
9
+ ...getDefaultOptions("QyKVDataSaver_Worker"),
10
+ ...a
11
+ }), Object.assign(this, {
12
+ kvFolder: e,
13
+ aclFolder: path.join(e, "acl"),
14
+ qyAclSaver: new QyAclSaver(this, a),
15
+ qySnapshotSaver: new QySnapshotSaver(this, e, a)
16
+ });
17
+ }
18
+ start(e, a, s, t) {
19
+ var {
20
+ qyAclSaver: r,
21
+ qySnapshotSaver: n
22
+ } = this;
23
+ Object.assign(this, {
24
+ maxChangeId: e,
25
+ snapshotMaxChangeId: e,
26
+ currentAclNum: (s || a) + 1
27
+ }), r.start(this._getAclFilePath()), n.start(e, s, t);
28
+ }
29
+ async stop() {
30
+ var {
31
+ qyAclSaver: e,
32
+ options: a,
33
+ qySnapshotSaver: s,
34
+ currentAclNum: t,
35
+ maxChangeId: r,
36
+ snapshotMaxChangeId: n
37
+ } = this;
38
+ this.isStopping = !0, await e.stop(), a.saveSnapshotAtStop && n < r && (this.isSavingSnapshot && await new Promise(e => this.saveSnapshotResolve = e),
39
+ logger.log(`Saving snapshot ${t} at stop.`), s.saveSnapshot(t)), await s.stop();
40
+ }
41
+ callSave(e, a, s) {
42
+ return this._save(e, a, s, !0);
43
+ }
44
+ castSave(e, a, s) {
45
+ return this._save(e, a, s);
46
+ }
47
+ releaseSnapshot(e) {
48
+ this.qySnapshotSaver.releaseSnapshot(e);
49
+ }
50
+ _save(e, a, s, t) {
51
+ var {
52
+ qyAclSaver: r,
53
+ qySnapshotSaver: n,
54
+ options: o,
55
+ snapshotMaxChangeId: h,
56
+ currentAclNum: i,
57
+ isSavingSnapshot: S
58
+ } = this;
59
+ let l;
60
+ return t ? l = r.callSave(e, a) : r.castSave(e, a), this._removeSyncedKeys(a, s),
61
+ n.castSave(e, a, s), this.maxChangeId = e, !S && e - h >= o.snapshotMaxChangeCount && (++this.currentAclNum,
62
+ r.switch(this._getAclFilePath()), n.saveSnapshot(i), Object.assign(this, {
63
+ snapshotMaxChangeId: e,
64
+ isSavingSnapshot: !0
65
+ })), l;
66
+ }
67
+ _removeSyncedKeys(a, s) {
68
+ if (s) {
69
+ var t = s.length;
70
+ for (let e = 0; e < t; e += 2) delCmdKey(a, s[e]);
71
+ }
72
+ }
73
+ _getAclFilePath() {
74
+ return path.join(this.aclFolder, this.currentAclNum + "_acl.txt");
75
+ }
76
+ onSnapshotSavedChangeId(e, a) {
77
+ this.isSavingSnapshot = !1, this.saveSnapshotResolve ? this.saveSnapshotResolve() : this.isStopping || this.castParent("onSnapshotSavedChangeId", e, a);
78
+ }
79
+ }
80
+
81
+ module.exports = QyKVDataSaver_Worker;
@@ -0,0 +1,14 @@
1
+ let QyMessager = require("./QyMessager.js").QyMessager, getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions;
2
+
3
+ class QyLogFileSaver extends QyMessager {
4
+ constructor(e, s) {
5
+ super(e, "QyLogFileSaver_Worker.js", s = {
6
+ ...getDefaultOptions("QyLogFileSaver"),
7
+ ...s
8
+ }, s, [ "switch" ], [ "save" ]);
9
+ }
10
+ }
11
+
12
+ Object.assign(module.exports, {
13
+ QyLogFileSaver: QyLogFileSaver
14
+ });
@@ -0,0 +1,26 @@
1
+ let getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QyBinWriter = require("./QyBinWriter.js").QyBinWriter;
2
+
3
+ class QyLogFileSaver_Worker extends QyMessageWorker {
4
+ constructor(e) {
5
+ super(e = {
6
+ ...getDefaultOptions("QyLogFileSaver_Worker"),
7
+ ...e
8
+ }), Object.assign(this, {
9
+ qyBinWriter: new QyBinWriter(e)
10
+ });
11
+ }
12
+ async start(e) {
13
+ await this.qyBinWriter.start(e);
14
+ }
15
+ async stop() {
16
+ await this.qyBinWriter.stop();
17
+ }
18
+ async switch(e) {
19
+ await this.qyBinWriter.switch(e);
20
+ }
21
+ save(e, r) {
22
+ this.qyBinWriter.save(e, r);
23
+ }
24
+ }
25
+
26
+ module.exports = QyLogFileSaver_Worker;
@@ -0,0 +1,76 @@
1
+ let fs = require("node:fs"), path = require("node:path"), Console = require("node:console").Console, {
2
+ isMainThread,
3
+ threadId
4
+ } = require("node:worker_threads"), ThreadMark = isMainThread ? "" : `W${threadId}:`, LOG_LEVEL = {
5
+ Info: 0,
6
+ Debug: 1,
7
+ Warn: 2,
8
+ Error: 3,
9
+ None: 4
10
+ }, strToLogLevel = {
11
+ verbose: LOG_LEVEL.Info,
12
+ info: LOG_LEVEL.Info,
13
+ log: LOG_LEVEL.Debug,
14
+ debug: LOG_LEVEL.Debug,
15
+ warn: LOG_LEVEL.Warn,
16
+ error: LOG_LEVEL.Error,
17
+ none: LOG_LEVEL.None
18
+ }, logLevelToStr = Object.fromEntries(Object.entries(strToLogLevel).map(([ e, o ]) => [ o, e ])), logger = {
19
+ currentLogLevel: LOG_LEVEL.Debug,
20
+ info: function() {
21
+ logger.currentLogLevel == LOG_LEVEL.Info && writeLog("info", arguments);
22
+ },
23
+ log: function() {
24
+ logger.currentLogLevel <= LOG_LEVEL.Debug && writeLog("log", arguments);
25
+ },
26
+ debug: function() {
27
+ logger.currentLogLevel <= LOG_LEVEL.Debug && writeLog("debug", arguments);
28
+ },
29
+ warn: function() {
30
+ logger.currentLogLevel <= LOG_LEVEL.Warn && writeLog("warn", arguments);
31
+ },
32
+ error: function() {
33
+ logger.currentLogLevel <= LOG_LEVEL.Error && writeLog("error", arguments);
34
+ }
35
+ }, fileConsole, stdout, stderr;
36
+
37
+ function closeFileConsole() {
38
+ fileConsole = void 0, stdout && (stdout.close(), stdout = void 0), stderr && (stderr.close(),
39
+ stderr = void 0);
40
+ }
41
+
42
+ function setFileConsoleDateDirPath(e, o) {
43
+ closeFileConsole();
44
+ o = {
45
+ flags: o ? "w" : "a",
46
+ flush: !0,
47
+ autoClose: !1
48
+ };
49
+ stdout = fs.createWriteStream(path.join(e, "stdout.txt"), o), stderr = fs.createWriteStream(path.join(e, "stderr.txt"), o),
50
+ fileConsole = new Console({
51
+ stdout: stdout,
52
+ stderr: stderr,
53
+ inspectOptions: {
54
+ depth: 3,
55
+ colors: !1
56
+ }
57
+ });
58
+ }
59
+
60
+ function writeLog(e, o) {
61
+ var r = new Error().stack.split("\n"), r = `${new Date().toISOString()} ${ThreadMark}${/([^/\\()]+)\)?$/.exec(r[3])[1]}:${e}:`;
62
+ console[e](r, ...o), fileConsole && fileConsole[e](r, ...o);
63
+ }
64
+
65
+ Object.defineProperty(logger, "level", {
66
+ get() {
67
+ return logLevelToStr[logger.currentLogLevel];
68
+ },
69
+ set(e) {
70
+ logger.currentLogLevel = strToLogLevel[e];
71
+ }
72
+ }), globalThis.logger = logger, Object.assign(module.exports, {
73
+ logger: logger,
74
+ setFileConsoleDateDirPath: setFileConsoleDateDirPath,
75
+ closeFileConsole: closeFileConsole
76
+ });