datafolder 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. package/README.md +169 -0
  2. package/index.js +3 -0
  3. package/package.json +27 -0
  4. package/sample.js +37 -0
  5. package/scripts/airbnb.js +73 -0
  6. package/scripts/airbnb2.js +74 -0
  7. package/scripts/airbnb2_queries.js +231 -0
  8. package/scripts/airbnb_queries.js +264 -0
  9. package/scripts/downloadFile.js +42 -0
  10. package/scripts/express.js +46 -0
  11. package/scripts/fastify.js +79 -0
  12. package/scripts/flights.js +75 -0
  13. package/scripts/flights2.js +81 -0
  14. package/scripts/flights2_queries.js +453 -0
  15. package/scripts/flights_queries.js +403 -0
  16. package/src/QyAcl.js +252 -0
  17. package/src/QyAclSaver.js +14 -0
  18. package/src/QyAclSaver_Worker.js +29 -0
  19. package/src/QyAclWriter.js +19 -0
  20. package/src/QyBatch.js +501 -0
  21. package/src/QyBinWriter.js +140 -0
  22. package/src/QyCD.js +61 -0
  23. package/src/QyCache.js +212 -0
  24. package/src/QyCompressor.js +60 -0
  25. package/src/QyCompressor_Worker.js +13 -0
  26. package/src/QyDB.js +122 -0
  27. package/src/QyDefaultOptions.js +59 -0
  28. package/src/QyDir.js +192 -0
  29. package/src/QyFetch.js +38 -0
  30. package/src/QyFile.js +124 -0
  31. package/src/QyFileContentUtils.js +175 -0
  32. package/src/QyFileLogger.js +58 -0
  33. package/src/QyIndex.js +237 -0
  34. package/src/QyIndexPropTree.js +92 -0
  35. package/src/QyIntersectNode.js +48 -0
  36. package/src/QyItem.js +105 -0
  37. package/src/QyJsonSaver.js +20 -0
  38. package/src/QyKVData.js +289 -0
  39. package/src/QyKVDataCleaner.js +17 -0
  40. package/src/QyKVDataCleaner_Worker.js +94 -0
  41. package/src/QyKVDataLoader.js +65 -0
  42. package/src/QyKVDataSaver.js +17 -0
  43. package/src/QyKVDataSaver_Worker.js +81 -0
  44. package/src/QyLogFileSaver.js +14 -0
  45. package/src/QyLogFileSaver_Worker.js +26 -0
  46. package/src/QyLogger.js +76 -0
  47. package/src/QyMessageWorker.js +41 -0
  48. package/src/QyMessager.js +84 -0
  49. package/src/QyMessager_Run.js +13 -0
  50. package/src/QyMod.js +15 -0
  51. package/src/QyOpRunner.js +73 -0
  52. package/src/QyQueue.js +60 -0
  53. package/src/QySaver.js +83 -0
  54. package/src/QySnapshotCompactor.js +17 -0
  55. package/src/QySnapshotCompactor_Worker.js +151 -0
  56. package/src/QySnapshotSaver.js +17 -0
  57. package/src/QySnapshotSaver_Worker.js +164 -0
  58. package/src/QySnapshots.js +354 -0
  59. package/src/QyStructure.js +50 -0
  60. package/src/QyTrigger.js +133 -0
  61. package/src/QyUnionNode.js +105 -0
  62. package/src/QyUtils.js +250 -0
  63. package/src/QyWorker.js +33 -0
  64. package/src/hackernews.js +93 -0
  65. package/src/rpc/evalCode.js +9 -0
  66. package/src/rpc/getFileContent.js +16 -0
  67. package/src/rpc/getSubItems.js +32 -0
  68. package/src/rpc/queryFiles.js +33 -0
  69. package/src/rpc/reloadRpc.js +1 -0
  70. package/src/simulate.js +114 -0
  71. package/src/simulate2.js +110 -0
  72. package/src/soc_pokec.js +43 -0
  73. package/static/bootstrap-5.3.8-dist/css/bootstrap.min.css +6 -0
  74. package/static/bootstrap-5.3.8-dist/js/bootstrap.bundle.js +6312 -0
  75. package/static/datafolderui/QyStyledElement.js +30 -0
  76. package/static/datafolderui/QyUIFolderTree.js +203 -0
  77. package/static/datafolderui/QyUIUtils.js +42 -0
  78. package/static/datafolderui/display-tree.js +108 -0
  79. package/static/datafolderui/explore-tree.js +122 -0
  80. package/static/datafolderui/folder-tree.js +5 -0
  81. package/static/datafolderui/hscrpt.js +25 -0
  82. package/static/datafolderui/index.css +6 -0
  83. package/static/datafolderui/index.html +20 -0
  84. package/static/datafolderui/json-view.js +74 -0
  85. package/static/datafolderui/kv-input.js +66 -0
  86. package/static/datafolderui/list-input.js +52 -0
  87. package/static/datafolderui/main-page.js +98 -0
  88. package/static/datafolderui/num-input.js +27 -0
  89. package/static/datafolderui/query-editor.js +229 -0
  90. package/static/datafolderui/querySelectorShadowRoot.js +24 -0
  91. package/static/datafolderui/resizable-input.js +67 -0
  92. package/static/favicon.ico +0 -0
  93. package/static/icons-1.13.1/font/bootstrap-icons.min.css +5 -0
  94. package/static/icons-1.13.1/font/fonts/bootstrap-icons.woff +0 -0
  95. package/static/icons-1.13.1/font/fonts/bootstrap-icons.woff2 +0 -0
  96. package/static/json-view/dist/jsonview.js +433 -0
  97. package/static/tree.js/tree.css +106 -0
  98. package/static/tree.js/tree.js +374 -0
  99. package/static/w2ui/dist/w2ui.min.css +2 -0
  100. package/static/w2ui/src/less/icons/icons.json +1 -0
  101. package/static/w2ui/src/less/icons/preview.html +33 -0
  102. package/static/w2ui/src/less/icons/readme.md +15 -0
  103. package/static/w2ui/src/less/icons/svg/box.svg +3 -0
  104. package/static/w2ui/src/less/icons/svg/check.svg +3 -0
  105. package/static/w2ui/src/less/icons/svg/colors.svg +3 -0
  106. package/static/w2ui/src/less/icons/svg/columns.svg +5 -0
  107. package/static/w2ui/src/less/icons/svg/cross.svg +4 -0
  108. package/static/w2ui/src/less/icons/svg/drop.svg +6 -0
  109. package/static/w2ui/src/less/icons/svg/empty.svg +3 -0
  110. package/static/w2ui/src/less/icons/svg/eye-dropper.svg +4 -0
  111. package/static/w2ui/src/less/icons/svg/info.svg +8 -0
  112. package/static/w2ui/src/less/icons/svg/paste.svg +4 -0
  113. package/static/w2ui/src/less/icons/svg/pencil.svg +5 -0
  114. package/static/w2ui/src/less/icons/svg/plus.svg +3 -0
  115. package/static/w2ui/src/less/icons/svg/reload.svg +5 -0
  116. package/static/w2ui/src/less/icons/svg/search.svg +3 -0
  117. package/static/w2ui/src/less/icons/svg/settings.svg +8 -0
  118. package/static/w2ui/src/less/icons/w2ui-font.css +36 -0
  119. package/static/w2ui/src/less/icons/w2ui-font.woff +0 -0
  120. package/static/w2ui/src/less/src/buttons.less +184 -0
  121. package/static/w2ui/src/less/src/common.less +541 -0
  122. package/static/w2ui/src/less/src/fields.less +302 -0
  123. package/static/w2ui/src/less/src/form.less +319 -0
  124. package/static/w2ui/src/less/src/grid.less +1057 -0
  125. package/static/w2ui/src/less/src/icons.less +37 -0
  126. package/static/w2ui/src/less/src/layout.less +86 -0
  127. package/static/w2ui/src/less/src/mixins.less +87 -0
  128. package/static/w2ui/src/less/src/popup.less +277 -0
  129. package/static/w2ui/src/less/src/sidebar.less +343 -0
  130. package/static/w2ui/src/less/src/tabs.less +145 -0
  131. package/static/w2ui/src/less/src/toolbar.less +219 -0
  132. package/static/w2ui/src/less/src/tooltip.less +948 -0
  133. package/static/w2ui/src/less/w2ui.less +196 -0
  134. package/static/w2ui/src/locale/az-az.json +114 -0
  135. package/static/w2ui/src/locale/ba-ba.json +114 -0
  136. package/static/w2ui/src/locale/bg-bg.json +114 -0
  137. package/static/w2ui/src/locale/ca-es.json +114 -0
  138. package/static/w2ui/src/locale/da-dk.json +114 -0
  139. package/static/w2ui/src/locale/de-de.json +114 -0
  140. package/static/w2ui/src/locale/en-gb.json +17 -0
  141. package/static/w2ui/src/locale/en-us.json +17 -0
  142. package/static/w2ui/src/locale/es-ar.json +115 -0
  143. package/static/w2ui/src/locale/es-cr.json +119 -0
  144. package/static/w2ui/src/locale/es-es.json +114 -0
  145. package/static/w2ui/src/locale/es-mx.json +114 -0
  146. package/static/w2ui/src/locale/fr-fr.json +114 -0
  147. package/static/w2ui/src/locale/gl-es.json +114 -0
  148. package/static/w2ui/src/locale/hr-hr.json +114 -0
  149. package/static/w2ui/src/locale/hu-hu.json +114 -0
  150. package/static/w2ui/src/locale/id-id.json +114 -0
  151. package/static/w2ui/src/locale/it-it.json +114 -0
  152. package/static/w2ui/src/locale/ja-jp.json +114 -0
  153. package/static/w2ui/src/locale/kk-kz.json +116 -0
  154. package/static/w2ui/src/locale/ko-kr.json +114 -0
  155. package/static/w2ui/src/locale/lt-lt.json +114 -0
  156. package/static/w2ui/src/locale/nl-nl.json +114 -0
  157. package/static/w2ui/src/locale/no-no.json +114 -0
  158. package/static/w2ui/src/locale/pl-pl.json +114 -0
  159. package/static/w2ui/src/locale/pt-br.json +114 -0
  160. package/static/w2ui/src/locale/ru-ru.json +116 -0
  161. package/static/w2ui/src/locale/sk-sk.json +114 -0
  162. package/static/w2ui/src/locale/sl-si.json +116 -0
  163. package/static/w2ui/src/locale/sr-rs.json +114 -0
  164. package/static/w2ui/src/locale/tr-tr.json +114 -0
  165. package/static/w2ui/src/locale/uk-ua.json +114 -0
  166. package/static/w2ui/src/locale/zh-cn.json +114 -0
  167. package/static/w2ui/src/locale/zh-tw.json +114 -0
  168. package/static/w2ui/src/query.js +583 -0
  169. package/static/w2ui/src/readme.md +11 -0
  170. package/static/w2ui/src/w2base.js +311 -0
  171. package/static/w2ui/src/w2compat.js +207 -0
  172. package/static/w2ui/src/w2field.js +1854 -0
  173. package/static/w2ui/src/w2form.js +2411 -0
  174. package/static/w2ui/src/w2grid.js +8916 -0
  175. package/static/w2ui/src/w2layout.js +1194 -0
  176. package/static/w2ui/src/w2locale.js +129 -0
  177. package/static/w2ui/src/w2popup.js +909 -0
  178. package/static/w2ui/src/w2sidebar.js +1769 -0
  179. package/static/w2ui/src/w2tabs.js +669 -0
  180. package/static/w2ui/src/w2toolbar.js +1180 -0
  181. package/static/w2ui/src/w2tooltip.js +3134 -0
  182. package/static/w2ui/src/w2utils.js +2457 -0
@@ -0,0 +1,164 @@
1
+ let path = require("node:path"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, QyMessageWorker = require("./QyMessageWorker.js").QyMessageWorker, QySnapshots = require("./QySnapshots.js").QySnapshots, QyKVDataCleaner = require("./QyKVDataCleaner.js").QyKVDataCleaner;
2
+
3
+ class QySnapshotSaver_Worker extends QyMessageWorker {
4
+ constructor({
5
+ kvFolder: a,
6
+ options: s
7
+ }) {
8
+ super(s = {
9
+ ...getDefaultOptions("QySnapshotSaver_Worker"),
10
+ ...s
11
+ }, [ "stop", "waitForSnapshotSaved" ]);
12
+ var e = path.join(a, "snapshot");
13
+ Object.assign(this, {
14
+ kvFolder: a,
15
+ snapshotFolder: e,
16
+ qySnapshots: new QySnapshots(e, s),
17
+ qyKVDataCleaner: new QyKVDataCleaner(this, a, s),
18
+ aclKeyValueMap: {}
19
+ });
20
+ }
21
+ start(a, s, e) {
22
+ this.maxChangeId = a, this.qyKVDataCleaner.start(), s && this.waitForSnapshotSaved(this._createMaxSnapshot(s, a, e));
23
+ }
24
+ castSave(a, s, e) {
25
+ var {
26
+ aclKeyValueMap: t,
27
+ qySnapshots: n
28
+ } = this;
29
+ n.applyCmdObj(s, t, e), this.maxChangeId = a;
30
+ }
31
+ async saveSnapshot(a) {
32
+ var {
33
+ maxChangeId: s,
34
+ aclKeyValueMap: e
35
+ } = this;
36
+ this.aclKeyValueMap = {}, this.waitForSnapshotSaved(this._createMaxSnapshot(a, s, e));
37
+ }
38
+ releaseSnapshot(a) {
39
+ var s, e = this.qySnapshots, {
40
+ numToSnapshotMap: t,
41
+ keyToSnapshotMap: n
42
+ } = e, t = t[a], a = t.keyValueMap;
43
+ for (s in delete t.keyValueMap, a) delete n[s];
44
+ e.delSnapshot(t), global.gc && global.gc();
45
+ }
46
+ _createMaxSnapshot(a, s, e) {
47
+ var t = this._processOutdates(), a = {
48
+ snapshotNum: a,
49
+ keyValueMap: e,
50
+ outdatedCount: 0,
51
+ info: {
52
+ maxChangeId: s,
53
+ pMVersion: 0,
54
+ vLVersion: 0
55
+ }
56
+ }, e = (this.qySnapshots.addMaxSnapshot(a), this._saveKeyValueMapAndSnapshotInfo(a));
57
+ return {
58
+ saveMaxSnapshotInfoPromise: e,
59
+ outdates: t
60
+ };
61
+ }
62
+ async _saveKeyValueMapAndSnapshotInfo(a) {
63
+ var s = this.qySnapshots, {
64
+ snapshotNum: a,
65
+ info: e
66
+ } = (await s.saveKeyValueMap(a), a);
67
+ return await s.saveSnapshotInfo(a, e), logger.log("Created snapshot " + a),
68
+ {
69
+ snapshotNum: a,
70
+ maxChangeId: e.maxChangeId
71
+ };
72
+ }
73
+ _processOutdates() {
74
+ var a, s = this.qySnapshots, {
75
+ emptySnapshotNumList: e,
76
+ combiningSnapshotList: t,
77
+ pMCompactSnapshotList: n
78
+ } = s.calOutdates(), o = {}, p = [], h = [], r = [];
79
+ for (a of n) {
80
+ var {
81
+ snapshotNum: i,
82
+ info: l
83
+ } = a;
84
+ p.push(s.savePosMap(a)), h.push([ i, l.pMVersion ]), o[i] = {
85
+ ...l
86
+ };
87
+ }
88
+ n = t.length;
89
+ if (0 < n) {
90
+ var u, S, y = s.keyToSnapshotMap, {
91
+ keyValueMap: M,
92
+ info: n,
93
+ snapshotNum: d
94
+ } = u = s.getMaxCombinedSnapshot(t);
95
+ for (S of t) if (u != S) {
96
+ var g, {
97
+ keyValueMap: v,
98
+ snapshotNum: m
99
+ } = S;
100
+ for (g in v) M[g] = v[g], y[g] = u;
101
+ s.delSnapshot(S), e.push(m);
102
+ }
103
+ p.push(s.saveKeyValueMap(u)), h.push([ d, n.pMVersion ]), r.push(d, n.vLVersion),
104
+ o[d] = {
105
+ ...n
106
+ };
107
+ }
108
+ return {
109
+ emptySnapshotNumList: e,
110
+ pMChangeList: h,
111
+ vLChange: r,
112
+ snapshotInfoMap: o,
113
+ promises: p
114
+ };
115
+ }
116
+ async _op_stop() {
117
+ var {
118
+ qySnapshots: a,
119
+ qyKVDataCleaner: s
120
+ } = this;
121
+ a.closeAllFDs(), await s.stop();
122
+ }
123
+ async _op_waitForSnapshotSaved({
124
+ saveMaxSnapshotInfoPromise: a,
125
+ outdates: s
126
+ }) {
127
+ var e, {
128
+ snapshotNum: a,
129
+ maxChangeId: t
130
+ } = await a, {
131
+ emptySnapshotNumList: s,
132
+ pMChangeList: n,
133
+ vLChange: o,
134
+ snapshotInfoMap: p,
135
+ promises: h
136
+ } = s, {
137
+ qyKVDataCleaner: r,
138
+ qySnapshots: i,
139
+ options: l
140
+ } = this;
141
+ for (e in r.cleanAclFiles(a), 0 < h.length && await Promise.all(h), h.length = 0,
142
+ p) h.push(i.saveSnapshotInfo(e, p[e]));
143
+ 0 < h.length && await Promise.all(h), 0 < s.length && r.removeSnapshotsFiles(s),
144
+ 0 < n.length && r.cleanPMFiles(n), 0 < o.length && r.cleanVLFiles(o[0], o[1]);
145
+ let u;
146
+ a = i.snapshotCount;
147
+ 0 < a && a >= l.maxInMemSnapshotCount && (delete (s = this._getMinSnapshot()).prefixPM,
148
+ delete (u = {
149
+ ...s
150
+ }).keyValueMap, delete s.posMap, s.isOut = !0, logger.info("Sending snapshot " + s.snapshotNum)),
151
+ this.castParent("onSnapshotSavedChangeId", t, u);
152
+ }
153
+ _getMinSnapshot() {
154
+ var a, s = this.qySnapshots.numToSnapshotMap;
155
+ let e;
156
+ for (a in s) {
157
+ var t = s[a];
158
+ (!e || e.snapshotNum > a) && (e = t);
159
+ }
160
+ return e;
161
+ }
162
+ }
163
+
164
+ module.exports = QySnapshotSaver_Worker;
@@ -0,0 +1,354 @@
1
+ let fs = require("node:fs"), fsPromises = require("node:fs/promises"), path = require("node:path"), getDefaultOptions = require("./QyDefaultOptions.js").getDefaultOptions, {
2
+ Hash_Key_Length,
3
+ folderOrFileExists,
4
+ listToMap,
5
+ isEmptyObj
6
+ } = require("./QyUtils.js"), QyBinWriter = require("./QyBinWriter.js").QyBinWriter, {
7
+ bufferToNums,
8
+ numsToBuffer,
9
+ applyAclChange,
10
+ iterCmdKeys
11
+ } = require("./QyAcl.js"), QyJsonSaver = require("./QyJsonSaver.js").QyJsonSaver, logger = require("./QyLogger.js").logger, isArray = Array.isArray;
12
+
13
+ class QySnapshots {
14
+ constructor(t, e) {
15
+ (e = {
16
+ ...getDefaultOptions("QySnapshots"),
17
+ ...e
18
+ }).vLOutdatedPercent = 2 * e.pMOutdatedPercent;
19
+ var a = path.join(t, "info");
20
+ Object.assign(this, {
21
+ snapshotFolder: t,
22
+ snapshotInfoFolder: a,
23
+ options: e,
24
+ qyJsonSaver: new QyJsonSaver(a, e),
25
+ numToSnapshotMap: {},
26
+ keyToSnapshotMap: {},
27
+ fDMap: {},
28
+ prefixMap: {},
29
+ snapshotMaxChangeId: 0,
30
+ maxSnapshotNum: 0,
31
+ snapshotCount: 0
32
+ });
33
+ }
34
+ async loadSnapshotInfos() {
35
+ let {
36
+ snapshotInfoFolder: t,
37
+ qyJsonSaver: o,
38
+ numToSnapshotMap: n
39
+ } = this;
40
+ if (await folderOrFileExists(t)) {
41
+ var e = (await fsPromises.readdir(t)).filter(t => /^[1-9]/.test(t)).map(t => parseInt(t)).sort((t, e) => t - e);
42
+ let r = 0, s = 0;
43
+ await Promise.all(e.map(async t => {
44
+ var e = await o.loadFromFile(t + ".json"), a = n[t] = {
45
+ snapshotNum: t,
46
+ info: e,
47
+ posMap: {},
48
+ outdatedCount: 0
49
+ };
50
+ ++this.snapshotCount, t > r && (r = t, s = e.maxChangeId), await this._loadPrefixPM(a);
51
+ })), Object.assign(this, {
52
+ maxSnapshotNum: r,
53
+ snapshotMaxChangeId: s
54
+ }), this._setupPrefixMaps(e);
55
+ }
56
+ }
57
+ closeAllFDs() {
58
+ var t, e = this.fDMap;
59
+ for (t in e) fs.close(e[t]), delete e[t];
60
+ }
61
+ saveKeyValueMap(t) {
62
+ var e, {
63
+ keyValueMap: a,
64
+ info: r
65
+ } = t, s = (++r.vLVersion, new QyBinWriter()), o = (s.start(this.getVLFilePath(t)),
66
+ Object.keys(a).sort()), n = {};
67
+ let i = 0, p = 0;
68
+ for (e of o) {
69
+ var u = a[e];
70
+ let t;
71
+ null == u ? t = Undefined_Mark : (u = Buffer.isBuffer(u) ? u : _valueToBuffer(e, u),
72
+ u = (s.save(u), u).length, t = [ i, u ], i += u), n[e] = t, ++p;
73
+ }
74
+ return Object.assign(t, {
75
+ posMap: n,
76
+ outdatedCount: 0
77
+ }), Object.assign(r, {
78
+ totalCount: p,
79
+ outdatedCount: 0
80
+ }), Promise.all([ this.savePosMap(t, o), s.stop() ]);
81
+ }
82
+ addMaxSnapshot(t) {
83
+ var e, {
84
+ numToSnapshotMap: a,
85
+ keyToSnapshotMap: r
86
+ } = this, {
87
+ posMap: s,
88
+ keyValueMap: o,
89
+ snapshotNum: n,
90
+ info: i
91
+ } = t, i = i.maxChangeId, i = (Object.assign(this, {
92
+ maxSnapshotNum: n,
93
+ snapshotMaxChangeId: i
94
+ }), o || s);
95
+ for (e in i) r[e] = t;
96
+ a[n] = t, ++this.snapshotCount;
97
+ }
98
+ removeKey(t) {
99
+ var e = this.keyToSnapshotMap, a = e[t];
100
+ if (a) return this.removeKeyFromSnapshot(a, t), delete e[t], !0;
101
+ }
102
+ applyCmdObj(t, e, a) {
103
+ if (null != a) {
104
+ var r = a.length;
105
+ for (let t = 0; t < r; t += 2) {
106
+ var s = a[t], o = a[t + 1];
107
+ e[s] = null != o && _isListKey(s) ? listToMap(o) : o;
108
+ }
109
+ }
110
+ iterCmdKeys(t, t => {
111
+ t in e || (e[t] = this.getValueSync(t)), this.removeKey(t);
112
+ }), applyAclChange(e, t);
113
+ }
114
+ getValueSync(t) {
115
+ var e, a, r = this._getSnapshotByKey(t);
116
+ if (r) return {
117
+ keyValueMap: a,
118
+ posMap: e
119
+ } = r, a ? a[t] : (a = e[t]) && ([ e, a ] = a, 0 != a) ? this._readValFromFile(r, t, e, a) : void 0;
120
+ }
121
+ iterPMBuffer(e, a) {
122
+ var r = e.length;
123
+ for (let t = 0; t < r; ) {
124
+ var s = t + Hash_Key_Length, o = e.toString("latin1", t, s), n = [];
125
+ t = bufferToNums(e, s, n), a(o, n);
126
+ }
127
+ }
128
+ loadPartialPosMapBuffer(a, t, e) {
129
+ let {
130
+ posMap: r,
131
+ prefixPM: s
132
+ } = a, o = (delete s[t], this).keyToSnapshotMap;
133
+ this.iterPMBuffer(e, (t, e) => {
134
+ t in o ? ++a.outdatedCount : (r[t] = e, o[t] = a);
135
+ });
136
+ }
137
+ delSnapshot(t) {
138
+ delete this.numToSnapshotMap[t.snapshotNum], --this.snapshotCount, this._closeFD(this.getVLFilePath(t)),
139
+ this._closeFD(this.getPMFilePath(t));
140
+ }
141
+ calOutdates() {
142
+ var t, e = [], a = [], r = [], s = [], {
143
+ options: o,
144
+ numToSnapshotMap: n
145
+ } = this, {
146
+ pMOutdatedPercent: i,
147
+ vLOutdatedPercent: p
148
+ } = o;
149
+ for (t in n) {
150
+ var u, h = n[t];
151
+ h.isOut || (100 == (u = _getTotalOutdatedPercent(h)) ? (this.delSnapshot(h),
152
+ e.push(t)) : p < u ? (this._canCombine(h) ? a : s).push(h) : _getCurrentOutdatedPercent(h) > i ? r.push(h) : this._isSmallSnapshot(h) && this._canCombine(h) && a.push(h));
153
+ }
154
+ return 1 == a.length && _getTotalOutdatedPercent(a[0]) < p && (a.length = 0),
155
+ 0 < r.length && logger.log("pMCompacts:" + r.map(t => t.snapshotNum)), 0 < s.length && logger.log("vLCompacts:" + s.map(t => t.snapshotNum)),
156
+ 0 < a.length && logger.log("combiningSnapshots:" + a.map(t => t.snapshotNum)),
157
+ {
158
+ emptySnapshotNumList: e,
159
+ combiningSnapshotList: a,
160
+ pMCompactSnapshotList: r,
161
+ vLCompactSnapshotList: s
162
+ };
163
+ }
164
+ getMaxCombinedSnapshot(t) {
165
+ let e, a;
166
+ for (var r of t) {
167
+ var s = r.snapshotNum;
168
+ (!a || a < s) && (e = r, a = s);
169
+ }
170
+ return e;
171
+ }
172
+ getPMFilePath(t) {
173
+ var {
174
+ snapshotNum: t,
175
+ info: e
176
+ } = t;
177
+ return path.join(this.snapshotFolder, "" + t, e.pMVersion + "_pM.txt");
178
+ }
179
+ getVLFilePath(t) {
180
+ var {
181
+ snapshotNum: t,
182
+ info: e
183
+ } = t;
184
+ return path.join(this.snapshotFolder, "" + t, e.vLVersion + "_vL.txt");
185
+ }
186
+ saveSnapshotInfo(t, e) {
187
+ var {
188
+ outdatedCount: a,
189
+ totalCount: r
190
+ } = e;
191
+ return e.outdated = Math.round(100 * a / r) + "%", this.qyJsonSaver.saveToFile(e, t + ".json", !0);
192
+ }
193
+ savePosMap(t, e) {
194
+ var a, {
195
+ outdatedCount: r,
196
+ info: s,
197
+ posMap: o
198
+ } = t, n = (s.outdatedCount += r, t.outdatedCount = 0, ++s.pMVersion, new QyBinWriter()), i = (n.start(this.getPMFilePath(t)),
199
+ t.prefixPM = {});
200
+ let p = 0, u = 0, h;
201
+ for (a of e = e || Object.keys(o).sort()) {
202
+ null == h ? h = _getPrefix(a) : a.startsWith(h) || (i[h] = [ p, u ],
203
+ h = _getPrefix(a), p += u, u = 0);
204
+ var l = Buffer.from(a), f = numsToBuffer(o[a]);
205
+ n.save(l), n.save(f), u += l.length + f.length;
206
+ }
207
+ return null != h && (i[h] = [ p, u ]), Promise.all([ n.stop(), this._savePrefixPM(t) ]);
208
+ }
209
+ isEmptySnapshot(t) {
210
+ var {
211
+ info: t,
212
+ outdatedCount: e
213
+ } = t;
214
+ return e + t.outdatedCount == t.totalCount;
215
+ }
216
+ _setupPrefixMaps(t) {
217
+ var e, {
218
+ numToSnapshotMap: a,
219
+ prefixMap: r
220
+ } = this;
221
+ for (e of t) {
222
+ var s, o = a[e], n = o.prefixPM;
223
+ for (s in n) {
224
+ var i = r[s];
225
+ i ? i.push(o) : r[s] = [ o ];
226
+ }
227
+ }
228
+ }
229
+ async _loadPrefixPM(t) {
230
+ var e = this._getPrefixPMFilePath(t), a = (logger.info("Loading " + e),
231
+ await fsPromises.readFile(e)), r = a.length, s = t.prefixPM = {};
232
+ for (let t = 0; t < r; ) {
233
+ var o = t + Prefix_Length, n = s[a.toString("latin1", t, o)] = [];
234
+ t = bufferToNums(a, o, n);
235
+ }
236
+ }
237
+ _loadPartialPosMap(t, e) {
238
+ var a, {
239
+ posMap: r,
240
+ prefixPM: s
241
+ } = t, s = s[e];
242
+ if (s) return [ s, a ] = s, s = this._readBufferFromFile(this.getPMFilePath(t), s, a),
243
+ this.loadPartialPosMapBuffer(t, e, s), r;
244
+ }
245
+ _getSnapshotByKey(t) {
246
+ var {
247
+ keyToSnapshotMap: e,
248
+ prefixMap: a
249
+ } = this;
250
+ let r = e[t];
251
+ if (r) return r;
252
+ var s = _getPrefix(t), o = a[s];
253
+ if (o) for (;0 < o.length; ) {
254
+ r = o.pop();
255
+ var n = this._loadPartialPosMap(r, s);
256
+ if (n && t in n) return r;
257
+ }
258
+ }
259
+ _readValFromFile(t, e, a, r) {
260
+ t = this.getVLFilePath(t), t = this._readBufferFromFile(t, a, r);
261
+ if (t) return _parseBuffer(e, t);
262
+ }
263
+ _readBufferFromFile(e, a, r) {
264
+ var s = this._getFD(e), o = Buffer.allocUnsafe(r);
265
+ for (let t = 0; t < r; ) {
266
+ var n = fs.readSync(s, o, t, r - t, a + t);
267
+ if (0 == n) return void logger.error(`readSync ${e} at ${a + t} returning 0 byte`);
268
+ t += n;
269
+ }
270
+ return o;
271
+ }
272
+ _canCombine(t) {
273
+ var {
274
+ keyValueMap: e,
275
+ prefixPM: a
276
+ } = t;
277
+ return e || null == a || isEmptyObj(t.prefixPM);
278
+ }
279
+ _isSmallSnapshot(t) {
280
+ return t.info.totalCount < this.options.minSnapshotKeyCount;
281
+ }
282
+ _getPrefixPMFilePath(t) {
283
+ var {
284
+ snapshotNum: t,
285
+ info: e
286
+ } = t;
287
+ return path.join(this.snapshotFolder, "" + t, e.pMVersion + "_prefixPM.txt");
288
+ }
289
+ removeKeyFromSnapshot(t, e) {
290
+ var {
291
+ posMap: a,
292
+ keyValueMap: r
293
+ } = t;
294
+ r && delete r[e], a && delete a[e], ++t.outdatedCount;
295
+ }
296
+ _getFD(e) {
297
+ var t = this.fDMap;
298
+ let a = t[e];
299
+ if (!a) {
300
+ logger.info("Opening " + e);
301
+ try {
302
+ a = t[e] = fs.openSync(e, "r");
303
+ } catch (t) {
304
+ throw logger.error(`Opening ${e} failed:`, t), t;
305
+ }
306
+ }
307
+ return a;
308
+ }
309
+ _closeFD(t) {
310
+ var e = this.fDMap, a = e[t];
311
+ null != a && (fs.close(a), delete e[t]);
312
+ }
313
+ _savePrefixPM(t) {
314
+ var e, a = new QyBinWriter(), r = (a.start(this._getPrefixPMFilePath(t)),
315
+ t).prefixPM;
316
+ for (e in r) a.save(Buffer.from(e)), a.save(numsToBuffer(r[e]));
317
+ return a.stop();
318
+ }
319
+ }
320
+
321
+ let Undefined_Mark = [ 0, 0 ], Prefix_Length = 3;
322
+
323
+ function _getPrefix(t) {
324
+ return t.substring(0, Prefix_Length);
325
+ }
326
+
327
+ function _isListKey(t) {
328
+ return !t.startsWith("c");
329
+ }
330
+
331
+ function _valueToBuffer(t, e) {
332
+ return Buffer.from(JSON.stringify(_isListKey(t) && !isArray(e) ? Object.keys(e) : e) + "\n");
333
+ }
334
+
335
+ function _parseBuffer(t, e) {
336
+ e = JSON.parse(e);
337
+ return _isListKey(t) ? listToMap(e) : e;
338
+ }
339
+
340
+ function _getTotalOutdatedPercent(t) {
341
+ var {
342
+ outdatedCount: t,
343
+ info: e
344
+ } = t, a = e.totalCount, t = t + e.outdatedCount;
345
+ return t == a ? 100 : Math.round(t / a * 100);
346
+ }
347
+
348
+ function _getCurrentOutdatedPercent(t) {
349
+ return Math.round(t.outdatedCount / t.info.totalCount * 100);
350
+ }
351
+
352
+ Object.assign(module.exports, {
353
+ QySnapshots: QySnapshots
354
+ });
@@ -0,0 +1,50 @@
1
+ let {
2
+ readdir,
3
+ rm,
4
+ writeFile,
5
+ rename
6
+ } = require("node:fs/promises"), path = require("node:path"), ensureDir = require("./QyUtils.js").ensureDir;
7
+
8
+ async function dumpStructure(e, r) {
9
+ let t = {};
10
+ for (var i of r.subdirList) {
11
+ let e = i.name;
12
+ for (;r.getFile(e); ) e += "_d";
13
+ e = cleanName(e), t[e] = {
14
+ x: i,
15
+ count: 0
16
+ };
17
+ }
18
+ await ensureDir(e);
19
+ var a, n = [], u = [];
20
+ let o = {};
21
+ for (a of await readdir(e, {
22
+ withFileTypes: !0
23
+ })) {
24
+ var s = a.name;
25
+ a.isFile() ? n.push(s) : (u.push(s), o[s] = 1);
26
+ }
27
+ var l, p, m, c, h, f = [];
28
+ for (l of u) t[l] || ((p = d((p = l) + "_d") || d(p.replace(/_d$/, ""))) ? (f.push(rename(path.join(e, l), path.join(e, p))),
29
+ t[p].count = 1) : f.push(rm(path.join(e, l), {
30
+ recursive: !0
31
+ })));
32
+ for (m of n) r.getFile(m) || f.push(rm(path.join(e, m)));
33
+ await Promise.all(f), f.length = 0;
34
+ for (c of r.fileList) f.push(writeFile(path.join(e, cleanName(c.name)), c.contentAsText, {
35
+ flush: !0
36
+ }));
37
+ for (h in await Promise.all(f), f.length = 0, t) f.push(dumpStructure(path.join(e, h), t[h].x));
38
+ function d(e) {
39
+ return e && 0 == t[e]?.count && !o[e] ? e : void 0;
40
+ }
41
+ await Promise.all(f);
42
+ }
43
+
44
+ function cleanName(e) {
45
+ return e.replace(/"/g, "");
46
+ }
47
+
48
+ Object.assign(module.exports, {
49
+ dumpStructure: dumpStructure
50
+ });
@@ -0,0 +1,133 @@
1
+ let util = require("node:util"), {
2
+ getValByNamePath,
3
+ setValByNamePath
4
+ } = require("./QyFileContentUtils.js"), {
5
+ isEmptyObj,
6
+ isMatchName,
7
+ getMatchParam,
8
+ arrayLast,
9
+ propNameJoin
10
+ } = require("./QyUtils.js"), isArray = Array.isArray;
11
+
12
+ class QyTriggerNode {
13
+ constructor(e, r) {
14
+ Object.assign(this, {
15
+ name: e,
16
+ parent: r,
17
+ children: {},
18
+ triggerMap: {}
19
+ });
20
+ }
21
+ insertTrigger(e, r, t, a) {
22
+ let i = this;
23
+ for (var g of r) {
24
+ isMatchName(g) && (g = "*");
25
+ var s = i.children;
26
+ i = s[g] || (s[g] = new QyTriggerNode(g, i));
27
+ }
28
+ var l = i.triggerMap, o = l[e];
29
+ o ? Object.assign(o, {
30
+ triggerName: e,
31
+ fileNamePath: r,
32
+ propNamePathList: t,
33
+ fun: a
34
+ }) : l[e] = new QyTrigger(e, r, t, a);
35
+ }
36
+ removeTrigger(e, r) {
37
+ let t = this;
38
+ for (var a of r) if (isMatchName(a) && (a = "*"), !(t = t.children[a])) return !1;
39
+ r = t.triggerMap;
40
+ return !!r[e] && (delete r[e], t.isEmpty && t.removeEmptyNodes(), !0);
41
+ }
42
+ get isEmpty() {
43
+ return isEmptyObj(this.children) && isEmptyObj(this.triggerMap);
44
+ }
45
+ removeEmptyNodes() {
46
+ let e = this;
47
+ for (;;) {
48
+ var {
49
+ parent: r,
50
+ name: t
51
+ } = e;
52
+ if (!r) return;
53
+ if (delete r.children[t], !r.isEmpty) return;
54
+ e = r;
55
+ }
56
+ }
57
+ checkTrigger(e, r) {
58
+ var t, a = this.triggerMap;
59
+ for (t in a) a[t].checkTrigger(e, r);
60
+ }
61
+ }
62
+
63
+ class QyTrigger {
64
+ constructor(e, r, t, a) {
65
+ Object.assign(this, {
66
+ triggerName: e,
67
+ fileNamePath: r,
68
+ propNamePathList: t,
69
+ fun: a
70
+ });
71
+ }
72
+ checkTrigger(e, r) {
73
+ var t, {
74
+ triggerName: a,
75
+ fileNamePath: i,
76
+ propNamePathList: g,
77
+ fun: s
78
+ } = this, l = e.fileContent;
79
+ let o;
80
+ for (t of g) {
81
+ var n = getValByNamePath(r.oldValue, t), h = getValByNamePath(r.newValue, t);
82
+ util.isDeepStrictEqual(n, h) || (n = {
83
+ oldValue: n,
84
+ newValue: h
85
+ }, (isArray(h) || isArray(getValByNamePath(l, t))) && (n.isArrayDelta = !0),
86
+ (o = o || {
87
+ _file: e,
88
+ _fileContent: l
89
+ })[propNameJoin(t)] = n, setValByNamePath(o, t, n));
90
+ }
91
+ if (o) {
92
+ var {
93
+ namePath: m,
94
+ qyCache: g
95
+ } = e;
96
+ for (let e = 0; e < m.length; ++e) {
97
+ var y = getMatchParam(i[e]);
98
+ y && (o[y] = m[e]);
99
+ }
100
+ try {
101
+ s && s(o), g.emit(a, o);
102
+ } catch (e) {
103
+ logger.error(`Running trigger ${a} failed:`, o, e);
104
+ }
105
+ }
106
+ }
107
+ }
108
+
109
+ let emptyArray = [];
110
+
111
+ function _getMatchedSubItems(e, r) {
112
+ return e ? isMatchName(r) ? Object.values(e) : (e = e[r]) ? [ e ] : emptyArray : emptyArray;
113
+ }
114
+
115
+ function updateTriggerNodes(e, a) {
116
+ if (0 != a.length) {
117
+ let r = [ e ], t = [];
118
+ for (let e = 0; e < a.length - 1; ++e) {
119
+ var i, g = a[e];
120
+ for (i of r) for (var s of _getMatchedSubItems(i.subdirMap, g)) i.setChildTriggerNodes(s),
121
+ t.push(s);
122
+ if (0 == t.length) return;
123
+ [ r, t ] = [ t, r ], t.length = 0;
124
+ }
125
+ var l, o = arrayLast(a);
126
+ for (l of r) for (var n of _getMatchedSubItems(l.fileMap, o)) l.setChildTriggerNodes(n);
127
+ }
128
+ }
129
+
130
+ Object.assign(module.exports, {
131
+ QyTriggerNode: QyTriggerNode,
132
+ updateTriggerNodes: updateTriggerNodes
133
+ });