@inlang/sdk 0.35.4 → 0.35.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapter/solidAdapter.test.js +1 -1
- package/dist/api.d.ts +2 -13
- package/dist/api.d.ts.map +1 -1
- package/dist/createMessagesQuery.d.ts.map +1 -1
- package/dist/createMessagesQuery.js +68 -82
- package/dist/createNewProject.test.js +1 -3
- package/dist/loadProject.d.ts.map +1 -1
- package/dist/loadProject.js +67 -32
- package/dist/loadProject.test.js +6 -2
- package/dist/persistence/batchedIO.d.ts +11 -0
- package/dist/persistence/batchedIO.d.ts.map +1 -0
- package/dist/persistence/batchedIO.js +49 -0
- package/dist/persistence/batchedIO.test.d.ts +2 -0
- package/dist/persistence/batchedIO.test.d.ts.map +1 -0
- package/dist/persistence/batchedIO.test.js +56 -0
- package/dist/persistence/filelock/acquireFileLock.d.ts.map +1 -1
- package/dist/persistence/filelock/acquireFileLock.js +3 -1
- package/dist/persistence/filelock/releaseLock.d.ts.map +1 -1
- package/dist/persistence/filelock/releaseLock.js +2 -1
- package/dist/persistence/store.d.ts +107 -0
- package/dist/persistence/store.d.ts.map +1 -0
- package/dist/persistence/store.js +99 -0
- package/dist/persistence/store.test.d.ts +2 -0
- package/dist/persistence/store.test.d.ts.map +1 -0
- package/dist/persistence/store.test.js +79 -0
- package/dist/persistence/storeApi.d.ts +22 -0
- package/dist/persistence/storeApi.d.ts.map +1 -0
- package/dist/persistence/storeApi.js +1 -0
- package/dist/reactivity/solid.test.js +1 -6
- package/dist/resolve-modules/plugins/resolvePlugins.d.ts.map +1 -1
- package/dist/resolve-modules/plugins/resolvePlugins.js +3 -10
- package/dist/test-utilities/sleep.d.ts +4 -0
- package/dist/test-utilities/sleep.d.ts.map +1 -0
- package/dist/test-utilities/sleep.js +9 -0
- package/dist/v2/helper.d.ts +131 -0
- package/dist/v2/helper.d.ts.map +1 -0
- package/dist/v2/helper.js +75 -0
- package/dist/v2/helper.test.d.ts +2 -0
- package/dist/v2/helper.test.d.ts.map +1 -0
- package/dist/v2/{createMessageBundle.test.js → helper.test.js} +1 -1
- package/dist/v2/index.d.ts +2 -0
- package/dist/v2/index.d.ts.map +1 -1
- package/dist/v2/index.js +2 -1
- package/dist/v2/mocks/index.d.ts +3 -0
- package/dist/v2/mocks/index.d.ts.map +1 -0
- package/dist/v2/mocks/index.js +2 -0
- package/dist/v2/mocks/multipleMatcher/bundle.d.ts +3 -0
- package/dist/v2/mocks/multipleMatcher/bundle.d.ts.map +1 -0
- package/dist/v2/mocks/multipleMatcher/bundle.js +194 -0
- package/dist/v2/mocks/multipleMatcher/bundle.test.d.ts +2 -0
- package/dist/v2/mocks/multipleMatcher/bundle.test.d.ts.map +1 -0
- package/dist/v2/mocks/multipleMatcher/bundle.test.js +10 -0
- package/dist/v2/mocks/plural/bundle.d.ts +1 -1
- package/dist/v2/mocks/plural/bundle.d.ts.map +1 -1
- package/dist/v2/mocks/plural/bundle.js +1 -1
- package/dist/v2/mocks/plural/bundle.test.js +6 -6
- package/dist/v2/shim.d.ts +12 -0
- package/dist/v2/shim.d.ts.map +1 -0
- package/dist/v2/shim.js +151 -0
- package/dist/v2/shim.test.d.ts +2 -0
- package/dist/v2/shim.test.d.ts.map +1 -0
- package/dist/v2/shim.test.js +49 -0
- package/dist/v2/stubQueryApi.d.ts +9 -0
- package/dist/v2/stubQueryApi.d.ts.map +1 -0
- package/dist/v2/stubQueryApi.js +38 -0
- package/dist/v2/types.d.ts +110 -0
- package/dist/v2/types.d.ts.map +1 -1
- package/dist/v2/types.js +9 -0
- package/package.json +9 -8
- package/src/adapter/solidAdapter.test.ts +1 -1
- package/src/api.ts +2 -13
- package/src/createMessagesQuery.ts +80 -99
- package/src/createNewProject.test.ts +1 -4
- package/src/loadProject.test.ts +6 -2
- package/src/loadProject.ts +86 -45
- package/src/persistence/batchedIO.test.ts +63 -0
- package/src/persistence/batchedIO.ts +64 -0
- package/src/persistence/filelock/acquireFileLock.ts +5 -2
- package/src/persistence/filelock/releaseLock.ts +2 -1
- package/src/persistence/store.test.ts +102 -0
- package/src/persistence/store.ts +119 -0
- package/src/persistence/storeApi.ts +19 -0
- package/src/reactivity/solid.test.ts +1 -8
- package/src/resolve-modules/plugins/resolvePlugins.ts +4 -13
- package/src/test-utilities/sleep.ts +11 -0
- package/src/v2/{createMessageBundle.test.ts → helper.test.ts} +1 -1
- package/src/v2/helper.ts +98 -0
- package/src/v2/index.ts +2 -0
- package/src/v2/mocks/index.ts +2 -0
- package/src/v2/mocks/multipleMatcher/bundle.test.ts +11 -0
- package/src/v2/mocks/multipleMatcher/bundle.ts +196 -0
- package/src/v2/mocks/plural/bundle.test.ts +6 -6
- package/src/v2/mocks/plural/bundle.ts +1 -1
- package/src/v2/shim.test.ts +56 -0
- package/src/v2/shim.ts +173 -0
- package/src/v2/stubQueryApi.ts +43 -0
- package/src/v2/types.ts +17 -0
- package/dist/persistence/plugin.d.ts +0 -31
- package/dist/persistence/plugin.d.ts.map +0 -1
- package/dist/persistence/plugin.js +0 -42
- package/dist/persistence/plugin.test.d.ts +0 -2
- package/dist/persistence/plugin.test.d.ts.map +0 -1
- package/dist/persistence/plugin.test.js +0 -49
- package/dist/v2/createMessageBundle.d.ts +0 -25
- package/dist/v2/createMessageBundle.d.ts.map +0 -1
- package/dist/v2/createMessageBundle.js +0 -36
- package/dist/v2/createMessageBundle.test.d.ts +0 -2
- package/dist/v2/createMessageBundle.test.d.ts.map +0 -1
- package/src/persistence/plugin.test.ts +0 -60
- package/src/persistence/plugin.ts +0 -56
- package/src/v2/createMessageBundle.ts +0 -43
|
@@ -174,7 +174,7 @@ describe("messages", () => {
|
|
|
174
174
|
project.setSettings({ ...project.settings(), languageTags: ["en"] });
|
|
175
175
|
// TODO: how can we await `setConfig` correctly
|
|
176
176
|
await new Promise((resolve) => setTimeout(resolve, 510));
|
|
177
|
-
expect(effectOnMessagesCounter).toBe(
|
|
177
|
+
expect(effectOnMessagesCounter).toBe(2); // 2 = setSetting (clearing the message index), subsequencial loadMessage call
|
|
178
178
|
expect(Object.values(project.query.messages.getAll()).length).toBe(2);
|
|
179
179
|
});
|
|
180
180
|
it("should react to message udpate", async () => {
|
package/dist/api.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ import type * as RuntimeError from "./errors.js";
|
|
|
3
3
|
import type * as ModuleResolutionError from "./resolve-modules/errors.js";
|
|
4
4
|
import type { MessageLintLevel, MessageLintRule, Message, Plugin, ProjectSettings, MessageLintReport } from "./versionedInterfaces.js";
|
|
5
5
|
import type { ResolvedPluginApi } from "./resolve-modules/plugins/types.js";
|
|
6
|
-
import type
|
|
6
|
+
import type { StoreApi } from "./persistence/storeApi.js";
|
|
7
7
|
export type InstalledPlugin = {
|
|
8
8
|
id: Plugin["id"];
|
|
9
9
|
displayName: Plugin["displayName"];
|
|
@@ -42,18 +42,8 @@ export type InlangProject = {
|
|
|
42
42
|
messages: MessageQueryApi;
|
|
43
43
|
messageLintReports: MessageLintReportsQueryApi;
|
|
44
44
|
};
|
|
45
|
-
|
|
46
|
-
messages?: Query<V2.Message>;
|
|
47
|
-
variants?: Query<V2.Variant>;
|
|
45
|
+
store?: StoreApi;
|
|
48
46
|
};
|
|
49
|
-
/**
|
|
50
|
-
* WIP template for async V2 crud interfaces
|
|
51
|
-
* E.g. `await project.messageBundles.get({ id: "..." })`
|
|
52
|
-
**/
|
|
53
|
-
interface Query<T> {
|
|
54
|
-
get: (args: unknown) => Promise<T>;
|
|
55
|
-
getAll: () => Promise<T[]>;
|
|
56
|
-
}
|
|
57
47
|
export type Subscribable<Value> = {
|
|
58
48
|
(): Value;
|
|
59
49
|
subscribe: (callback: (value: Value) => void) => void;
|
|
@@ -120,5 +110,4 @@ export type MessageLintReportsQueryApi = {
|
|
|
120
110
|
}, callback: (MessageLintRules: Readonly<MessageLintReport[]>) => void) => void;
|
|
121
111
|
};
|
|
122
112
|
};
|
|
123
|
-
export {};
|
|
124
113
|
//# sourceMappingURL=api.d.ts.map
|
package/dist/api.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,KAAK,YAAY,MAAM,aAAa,CAAA;AAChD,OAAO,KAAK,KAAK,qBAAqB,MAAM,6BAA6B,CAAA;AACzE,OAAO,KAAK,EACX,gBAAgB,EAChB,eAAe,EACf,OAAO,EACP,MAAM,EACN,eAAe,EACf,iBAAiB,EACjB,MAAM,0BAA0B,CAAA;AACjC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAA;AAC3E,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"api.d.ts","sourceRoot":"","sources":["../src/api.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAA;AAC5C,OAAO,KAAK,KAAK,YAAY,MAAM,aAAa,CAAA;AAChD,OAAO,KAAK,KAAK,qBAAqB,MAAM,6BAA6B,CAAA;AACzE,OAAO,KAAK,EACX,gBAAgB,EAChB,eAAe,EACf,OAAO,EACP,MAAM,EACN,eAAe,EACf,iBAAiB,EACjB,MAAM,0BAA0B,CAAA;AACjC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oCAAoC,CAAA;AAC3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAEzD,MAAM,MAAM,eAAe,GAAG;IAC7B,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;IAChB,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAA;IAClC,WAAW,EAAE,MAAM,CAAC,aAAa,CAAC,CAAA;IAClC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAA;CAExC,CAAA;AAED,MAAM,MAAM,wBAAwB,GAAG;IACtC,EAAE,EAAE,eAAe,CAAC,IAAI,CAAC,CAAA;IACzB,WAAW,EAAE,eAAe,CAAC,aAAa,CAAC,CAAA;IAC3C,WAAW,EAAE,eAAe,CAAC,aAAa,CAAC,CAAA;IAC3C;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,gBAAgB,CAAA;IACvB,cAAc,EAAE,eAAe,CAAC,gBAAgB,CAAC,CAAA;CACjD,CAAA;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B;;OAEG;IAEH,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,SAAS,EAAE;QACV,OAAO,EAAE,YAAY,CAAC,eAAe,EAAE,CAAC,CAAA;QACxC,gBAAgB,EAAE,YAAY,CAAC,wBAAwB,EAAE,CAAC,CAAA;KAC1D,CAAA;IACD,MAAM,EAAE,YAAY,CACnB,CAAC,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,OAAO,qBAAqB,CAAC,GAAG,KAAK,CAAC,EAAE,CAC9E,CAAA;IACD,SAAS,EAAE,YAAY,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC,CAAA;IACvD,QAAQ,EAAE,YAAY,CAAC,eAAe,CAAC,CAAA;IACvC,WAAW,EAAE,CAAC,MAAM,EAAE,eAAe,KAAK,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,2BAA2B,CAAC,CAAA;IAChG,KAAK,EAAE;QACN,QAAQ,EAAE,eAAe,CAAA;QACzB,kBAAkB,EAAE,0BAA0B,CAAA;KAC9C,CAAA;IAGD,KAAK,CAAC,EAAE,QAAQ,CAAA;CAChB,CAAA;AAMD,MAAM,MAAM,YAAY,CAAC,KAAK,IAAI;IACjC,IAAI,KAAK,CAAA;IACT,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,KAAK,IAAI,CAAA;CACrD,CAAA;AAED,MAAM,MAAM,oBAAoB,GAAG;IAClC,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;IACnF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;IACnF,eAAe,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;IACjE,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,KAAK,IAAI,CAAA;IACvC,SAAS,EAAE,MAAM,IAAI,CAAA;CACrB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC7B,MAAM,EAAE,CAAC,IAAI,EAAE;QAAE,IAAI,EAAE,OAAO,CAAA;KAAE,KAAK,OAAO,CAAA;IAC5C,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE;YAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE,CAAA;KAAE,KAAK,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG;QACtE,SAAS,EAAE,CACV,IAAI,EAAE;YAAE,KAAK,EAAE;gBAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;aAAE,CAAA;SAAE,EACtC,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,KAChC,IAAI,CAAA;KACT,CAAA;IAED,iBAAiB,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,CAAC,GAAG;QAChF,SAAS,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,KAAK,IAAI,CAAA;KAC7F,CAAA;IACD,kBAAkB,EAAE,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAIjD,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE;YAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,CAAA;KAAE,KAAK,OAAO,CAAA;IACnF,MAAM,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE;YAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,KAAK,IAAI,CAAA;IACvE,MAAM,EAAE,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE;YAAE,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;SAAE,CAAA;KAAE,KAAK,OAAO,CAAA;IAC3D,WAAW,EAAE,CAAC,QAAQ,EAAE,oBAAoB,GAAG,SAAS,EAAE,UAAU,EAAE,OAAO,KAAK,IAAI,CAAA;CACtF,CAAA;AAED,MAAM,MAAM,0BAA0B,GAAG;IACxC,MAAM,EAAE,YAAY,CAAC,iBAAiB,EAAE,CAAC,GAAG;QAC3C,OAAO,EAAE,MAAM,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAA;KAC3C,CAAA;IACD,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE;QACZ,KAAK,EAAE;YAAE,SAAS,EAAE,iBAAiB,CAAC,WAAW,CAAC,CAAA;SAAE,CAAA;KACpD,KAAK,QAAQ,CAAC,iBAAiB,EAAE,CAAC,CAAC,GAAG;QACtC,SAAS,EAAE,CACV,IAAI,EAAE;YAAE,KAAK,EAAE;gBAAE,SAAS,EAAE,iBAAiB,CAAC,WAAW,CAAC,CAAA;aAAE,CAAA;SAAE,EAC9D,QAAQ,EAAE,CAAC,gBAAgB,EAAE,QAAQ,CAAC,iBAAiB,EAAE,CAAC,KAAK,IAAI,KAC/D,IAAI,CAAA;KACT,CAAA;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createMessagesQuery.d.ts","sourceRoot":"","sources":["../src/createMessagesQuery.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAyC,MAAM,UAAU,CAAA;AAEpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AAEzE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAInD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;
|
|
1
|
+
{"version":3,"file":"createMessagesQuery.d.ts","sourceRoot":"","sources":["../src/createMessagesQuery.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAyC,MAAM,UAAU,CAAA;AAEpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AAEzE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAInD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAwB/D,KAAK,6BAA6B,GAAG;IACpC,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,iBAAiB,CAAA;IAC5B,QAAQ,EAAE,MAAM,eAAe,GAAG,SAAS,CAAA;IAC3C,eAAe,EAAE,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC,GAAG,SAAS,CAAA;IAC7E,0BAA0B,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAA;IAC/C,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAA;IACxC,mBAAmB,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,KAAK,IAAI,CAAA;CACxC,CAAA;AACD;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,EACnC,WAAW,EACX,SAAS,EACT,QAAQ,EACR,eAAe,EACf,0BAA0B,EAC1B,mBAAmB,EACnB,mBAAmB,GACnB,EAAE,6BAA6B,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC,CAmNpE"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ReactiveMap } from "./reactivity/map.js";
|
|
2
|
-
import { createEffect, onCleanup } from "./reactivity/solid.js";
|
|
2
|
+
import { createEffect, onCleanup, batch } from "./reactivity/solid.js";
|
|
3
3
|
import { createSubscribable } from "./loadProject.js";
|
|
4
4
|
import { createNodeishFsWithWatcher } from "./createNodeishFsWithWatcher.js";
|
|
5
5
|
import { stringifyMessage } from "./storage/helper.js";
|
|
@@ -9,9 +9,6 @@ import { releaseLock } from "./persistence/filelock/releaseLock.js";
|
|
|
9
9
|
import { PluginLoadMessagesError, PluginSaveMessagesError } from "./errors.js";
|
|
10
10
|
import { humanIdHash } from "./storage/human-id/human-readable-id.js";
|
|
11
11
|
const debug = _debug("sdk:messages");
|
|
12
|
-
function sleep(ms) {
|
|
13
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
14
|
-
}
|
|
15
12
|
/**
|
|
16
13
|
* Creates a reactive query API for messages.
|
|
17
14
|
*/
|
|
@@ -194,7 +191,6 @@ export function createMessagesQuery({ projectPath, nodeishFs, settings, resolved
|
|
|
194
191
|
// - json plugin exports into separate file per language.
|
|
195
192
|
// - saving a message in two different languages would lead to a write in de.json first
|
|
196
193
|
// - This will leads to a load of the messages and since en.json has not been saved yet the english variant in the message would get overritten with the old state again
|
|
197
|
-
const maxMessagesPerTick = 500;
|
|
198
194
|
/**
|
|
199
195
|
* Messsage that loads messages from a plugin - this method synchronizes with the saveMessage funciton.
|
|
200
196
|
* If a save is in progress loading will wait until saving is done. If another load kicks in during this load it will queue the
|
|
@@ -227,89 +223,79 @@ async function loadMessagesViaPlugin(fs, lockDirPath, messageState, messages, de
|
|
|
227
223
|
settings: settingsValue,
|
|
228
224
|
nodeishFs: fs,
|
|
229
225
|
}));
|
|
230
|
-
let loadedMessageCount = 0;
|
|
231
226
|
const deletedMessages = new Set(messages.keys());
|
|
232
|
-
|
|
233
|
-
const
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
deletedMessages.delete(currentMessages[0].id);
|
|
245
|
-
// update message in place - leave message id and alias untouched
|
|
246
|
-
loadedMessageClone.alias = {};
|
|
247
|
-
// TODO #1585 we have to map the id of the importedMessage to the alias and fill the id property with the id of the existing message - change when import mesage provides importedMessage.alias
|
|
248
|
-
if (experimentalAliases) {
|
|
249
|
-
loadedMessageClone.alias["default"] = loadedMessageClone.id;
|
|
227
|
+
batch(() => {
|
|
228
|
+
for (const loadedMessage of loadedMessages) {
|
|
229
|
+
const loadedMessageClone = structuredClone(loadedMessage);
|
|
230
|
+
const currentMessages = [...messages.values()]
|
|
231
|
+
// TODO #1585 here we match using the id to support legacy load message plugins - after we introduced import / export methods we will use importedMessage.alias
|
|
232
|
+
.filter((message) => (experimentalAliases ? message.alias["default"] : message.id) === loadedMessage.id);
|
|
233
|
+
if (currentMessages.length > 1) {
|
|
234
|
+
// NOTE: if we happen to find two messages witht the sam alias we throw for now
|
|
235
|
+
// - this could be the case if one edits the aliase manualy
|
|
236
|
+
throw new Error("more than one message with the same id or alias found ");
|
|
237
|
+
}
|
|
238
|
+
else if (currentMessages.length === 1) {
|
|
250
239
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- length has checked beforhand
|
|
251
|
-
|
|
240
|
+
deletedMessages.delete(currentMessages[0].id);
|
|
241
|
+
// update message in place - leave message id and alias untouched
|
|
242
|
+
loadedMessageClone.alias = {};
|
|
243
|
+
// TODO #1585 we have to map the id of the importedMessage to the alias and fill the id property with the id of the existing message - change when import mesage provides importedMessage.alias
|
|
244
|
+
if (experimentalAliases) {
|
|
245
|
+
loadedMessageClone.alias["default"] = loadedMessageClone.id;
|
|
246
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion -- length has checked beforhand
|
|
247
|
+
loadedMessageClone.id = currentMessages[0].id;
|
|
248
|
+
}
|
|
249
|
+
// NOTE stringifyMessage encodes messages independent from key order!
|
|
250
|
+
const importedEnecoded = stringifyMessage(loadedMessageClone);
|
|
251
|
+
// NOTE could use hash instead of the whole object JSON to save memory...
|
|
252
|
+
if (messageState.messageLoadHash[loadedMessageClone.id] === importedEnecoded) {
|
|
253
|
+
// debug("skipping upsert!")
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
// This logic is preventing cycles - could also be handled if update api had a parameter for who triggered update
|
|
257
|
+
// e.g. when FS was updated, we don't need to write back to FS
|
|
258
|
+
// update is synchronous, so update effect will be triggered immediately
|
|
259
|
+
// NOTE: this might trigger a save before we have the chance to delete - but since save is async and waits for the lock acquired by this method - its save to set the flags afterwards
|
|
260
|
+
messages.set(loadedMessageClone.id, loadedMessageClone);
|
|
261
|
+
// NOTE could use hash instead of the whole object JSON to save memory...
|
|
262
|
+
messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded;
|
|
263
|
+
delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone, [
|
|
264
|
+
...messages.values(),
|
|
265
|
+
]);
|
|
252
266
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
267
|
+
else {
|
|
268
|
+
// message with the given alias does not exist so far
|
|
269
|
+
loadedMessageClone.alias = {};
|
|
270
|
+
// TODO #1585 we have to map the id of the importedMessage to the alias - change when import mesage provides importedMessage.alias
|
|
271
|
+
if (experimentalAliases) {
|
|
272
|
+
loadedMessageClone.alias["default"] = loadedMessageClone.id;
|
|
273
|
+
let currentOffset = 0;
|
|
274
|
+
let messsageId;
|
|
275
|
+
do {
|
|
276
|
+
messsageId = humanIdHash(loadedMessageClone.id, currentOffset);
|
|
277
|
+
if (messages.get(messsageId)) {
|
|
278
|
+
currentOffset += 1;
|
|
279
|
+
messsageId = undefined;
|
|
280
|
+
}
|
|
281
|
+
} while (messsageId === undefined);
|
|
282
|
+
// create a humanId based on a hash of the alias
|
|
283
|
+
loadedMessageClone.id = messsageId;
|
|
284
|
+
}
|
|
285
|
+
const importedEnecoded = stringifyMessage(loadedMessageClone);
|
|
286
|
+
// we don't have to check - done before hand if (messages.has(loadedMessageClone.id)) return false
|
|
287
|
+
messages.set(loadedMessageClone.id, loadedMessageClone);
|
|
288
|
+
messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded;
|
|
289
|
+
delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone, [
|
|
290
|
+
...messages.values(),
|
|
291
|
+
]);
|
|
259
292
|
}
|
|
260
|
-
// This logic is preventing cycles - could also be handled if update api had a parameter for who triggered update
|
|
261
|
-
// e.g. when FS was updated, we don't need to write back to FS
|
|
262
|
-
// update is synchronous, so update effect will be triggered immediately
|
|
263
|
-
// NOTE: this might trigger a save before we have the chance to delete - but since save is async and waits for the lock acquired by this method - its save to set the flags afterwards
|
|
264
|
-
messages.set(loadedMessageClone.id, loadedMessageClone);
|
|
265
|
-
// NOTE could use hash instead of the whole object JSON to save memory...
|
|
266
|
-
messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded;
|
|
267
|
-
delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone, [...messages.values()]);
|
|
268
|
-
loadedMessageCount++;
|
|
269
293
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
// TODO #1585 we have to map the id of the importedMessage to the alias - change when import mesage provides importedMessage.alias
|
|
274
|
-
if (experimentalAliases) {
|
|
275
|
-
loadedMessageClone.alias["default"] = loadedMessageClone.id;
|
|
276
|
-
let currentOffset = 0;
|
|
277
|
-
let messsageId;
|
|
278
|
-
do {
|
|
279
|
-
messsageId = humanIdHash(loadedMessageClone.id, currentOffset);
|
|
280
|
-
if (messages.get(messsageId)) {
|
|
281
|
-
currentOffset += 1;
|
|
282
|
-
messsageId = undefined;
|
|
283
|
-
}
|
|
284
|
-
} while (messsageId === undefined);
|
|
285
|
-
// create a humanId based on a hash of the alias
|
|
286
|
-
loadedMessageClone.id = messsageId;
|
|
287
|
-
}
|
|
288
|
-
const importedEnecoded = stringifyMessage(loadedMessageClone);
|
|
289
|
-
// we don't have to check - done before hand if (messages.has(loadedMessageClone.id)) return false
|
|
290
|
-
messages.set(loadedMessageClone.id, loadedMessageClone);
|
|
291
|
-
messageState.messageLoadHash[loadedMessageClone.id] = importedEnecoded;
|
|
292
|
-
delegate?.onMessageUpdate(loadedMessageClone.id, loadedMessageClone, [...messages.values()]);
|
|
293
|
-
loadedMessageCount++;
|
|
294
|
+
for (const deletedMessageId of deletedMessages) {
|
|
295
|
+
messages.delete(deletedMessageId);
|
|
296
|
+
delegate?.onMessageDelete(deletedMessageId, [...messages.values()]);
|
|
294
297
|
}
|
|
295
|
-
|
|
296
|
-
// move loading of the next messages to the next ticks to allow solid to cleanup resources
|
|
297
|
-
// solid needs some time to settle and clean up
|
|
298
|
-
// https://github.com/solidjs-community/solid-primitives/blob/9ca76a47ffa2172770e075a90695cf933da0ff48/packages/trigger/src/index.ts#L64
|
|
299
|
-
await sleep(0);
|
|
300
|
-
loadedMessageCount = 0;
|
|
301
|
-
}
|
|
302
|
-
}
|
|
303
|
-
loadedMessageCount = 0;
|
|
304
|
-
for (const deletedMessageId of deletedMessages) {
|
|
305
|
-
messages.delete(deletedMessageId);
|
|
306
|
-
delegate?.onMessageDelete(deletedMessageId, [...messages.values()]);
|
|
307
|
-
loadedMessageCount++;
|
|
308
|
-
if (loadedMessageCount > maxMessagesPerTick) {
|
|
309
|
-
await sleep(0);
|
|
310
|
-
loadedMessageCount = 0;
|
|
311
|
-
}
|
|
312
|
-
}
|
|
298
|
+
});
|
|
313
299
|
await releaseLock(fs, lockDirPath, "loadMessage", lockTime);
|
|
314
300
|
lockTime = undefined;
|
|
315
301
|
debug("loadMessagesViaPlugin: " + loadedMessages.length + " Messages processed ");
|
|
@@ -4,9 +4,7 @@ import { mockRepo } from "@lix-js/client";
|
|
|
4
4
|
import { defaultProjectSettings } from "./defaultProjectSettings.js";
|
|
5
5
|
import { loadProject } from "./loadProject.js";
|
|
6
6
|
import { createMessage } from "./test-utilities/createMessage.js";
|
|
7
|
-
|
|
8
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
9
|
-
}
|
|
7
|
+
import { sleep } from "./test-utilities/sleep.js";
|
|
10
8
|
describe("createNewProject", () => {
|
|
11
9
|
it("should throw if a path does not end with .inlang", async () => {
|
|
12
10
|
const repo = await mockRepo();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"loadProject.d.ts","sourceRoot":"","sources":["../src/loadProject.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACX,aAAa,EAGb,YAAY,
|
|
1
|
+
{"version":3,"file":"loadProject.d.ts","sourceRoot":"","sources":["../src/loadProject.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EACX,aAAa,EAGb,YAAY,EAGZ,MAAM,UAAU,CAAA;AACjB,OAAO,EAAE,KAAK,cAAc,EAAkB,MAAM,4BAA4B,CAAA;AAkBhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAgBhD;;;;;;;GAOG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE;IACvC,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,UAAU,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,cAAc,CAAA;CACxB,GAAG,OAAO,CAAC,aAAa,CAAC,CAyQzB;AA+GD,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAQtE"}
|
package/dist/loadProject.js
CHANGED
|
@@ -14,6 +14,8 @@ import { maybeMigrateToDirectory } from "./migrations/migrateToDirectory.js";
|
|
|
14
14
|
import { maybeCreateFirstProjectId } from "./migrations/maybeCreateFirstProjectId.js";
|
|
15
15
|
import { capture } from "./telemetry/capture.js";
|
|
16
16
|
import { identifyProject } from "./telemetry/groupIdentify.js";
|
|
17
|
+
import { stubMessagesQuery, stubMessageLintReportsQuery } from "./v2/stubQueryApi.js";
|
|
18
|
+
import { openStore } from "./persistence/store.js";
|
|
17
19
|
import _debug from "debug";
|
|
18
20
|
const debug = _debug("sdk:loadProject");
|
|
19
21
|
const settingsCompiler = TypeCompiler.Compile(ProjectSettings);
|
|
@@ -47,8 +49,14 @@ export async function loadProject(args) {
|
|
|
47
49
|
// - if a repo is present, the project id will always be present
|
|
48
50
|
const { data: projectId } = await tryCatch(() => nodeishFs.readFile(args.projectPath + "/project_id", { encoding: "utf-8" }));
|
|
49
51
|
const [initialized, markInitAsComplete, markInitAsFailed] = createAwaitable();
|
|
52
|
+
const [loadedSettings, markSettingsAsLoaded, markSettingsAsFailed] = createAwaitable();
|
|
50
53
|
// -- settings ------------------------------------------------------------
|
|
51
54
|
const [settings, _setSettings] = createSignal();
|
|
55
|
+
let v2Persistence = false;
|
|
56
|
+
let locales = [];
|
|
57
|
+
// This effect currently has no signals
|
|
58
|
+
// TODO: replace createEffect with await loadSettings
|
|
59
|
+
// https://github.com/opral/inlang-message-sdk/issues/77
|
|
52
60
|
createEffect(() => {
|
|
53
61
|
// TODO:
|
|
54
62
|
// if (projectId) {
|
|
@@ -57,21 +65,23 @@ export async function loadProject(args) {
|
|
|
57
65
|
// })
|
|
58
66
|
// }
|
|
59
67
|
loadSettings({ settingsFilePath: projectPath + "/settings.json", nodeishFs })
|
|
60
|
-
.then((settings) =>
|
|
68
|
+
.then((settings) => {
|
|
69
|
+
setSettings(settings);
|
|
70
|
+
markSettingsAsLoaded();
|
|
71
|
+
})
|
|
61
72
|
.catch((err) => {
|
|
62
73
|
markInitAsFailed(err);
|
|
74
|
+
markSettingsAsFailed(err);
|
|
63
75
|
});
|
|
64
76
|
});
|
|
65
77
|
// TODO: create FS watcher and update settings on change
|
|
78
|
+
// https://github.com/opral/inlang-message-sdk/issues/35
|
|
66
79
|
const writeSettingsToDisk = skipFirst((settings) => _writeSettingsToDisk({ nodeishFs, settings, projectPath }));
|
|
67
80
|
const setSettings = (settings) => {
|
|
68
81
|
try {
|
|
69
82
|
const validatedSettings = parseSettings(settings);
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
pathPattern: projectPath + "/messages.json",
|
|
73
|
-
};
|
|
74
|
-
}
|
|
83
|
+
v2Persistence = !!validatedSettings.experimental?.persistence;
|
|
84
|
+
locales = validatedSettings.languageTags;
|
|
75
85
|
batch(() => {
|
|
76
86
|
// reset the resolved modules first - since they are no longer valid at that point
|
|
77
87
|
setResolvedModules(undefined);
|
|
@@ -99,32 +109,10 @@ export async function loadProject(args) {
|
|
|
99
109
|
})
|
|
100
110
|
.catch((err) => markInitAsFailed(err));
|
|
101
111
|
});
|
|
102
|
-
// -- messages ----------------------------------------------------------
|
|
103
|
-
let settingsValue;
|
|
104
|
-
createEffect(() => (settingsValue = settings())); // workaround to not run effects twice (e.g. settings change + modules change) (I'm sure there exists a solid way of doing this, but I haven't found it yet)
|
|
105
|
-
const [loadMessagesViaPluginError, setLoadMessagesViaPluginError] = createSignal();
|
|
106
|
-
const [saveMessagesViaPluginError, setSaveMessagesViaPluginError] = createSignal();
|
|
107
|
-
const messagesQuery = createMessagesQuery({
|
|
108
|
-
projectPath,
|
|
109
|
-
nodeishFs,
|
|
110
|
-
settings,
|
|
111
|
-
resolvedModules,
|
|
112
|
-
onInitialMessageLoadResult: (e) => {
|
|
113
|
-
if (e) {
|
|
114
|
-
markInitAsFailed(e);
|
|
115
|
-
}
|
|
116
|
-
else {
|
|
117
|
-
markInitAsComplete();
|
|
118
|
-
}
|
|
119
|
-
},
|
|
120
|
-
onLoadMessageResult: (e) => {
|
|
121
|
-
setLoadMessagesViaPluginError(e);
|
|
122
|
-
},
|
|
123
|
-
onSaveMessageResult: (e) => {
|
|
124
|
-
setSaveMessagesViaPluginError(e);
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
112
|
// -- installed items ----------------------------------------------------
|
|
113
|
+
let settingsValue;
|
|
114
|
+
// workaround to not run effects twice (e.g. settings change + modules change) (I'm sure there exists a solid way of doing this, but I haven't found it yet)
|
|
115
|
+
createEffect(() => (settingsValue = settings()));
|
|
128
116
|
const installedMessageLintRules = () => {
|
|
129
117
|
if (!resolvedModules())
|
|
130
118
|
return [];
|
|
@@ -151,9 +139,53 @@ export async function loadProject(args) {
|
|
|
151
139
|
settingsSchema: plugin.settingsSchema,
|
|
152
140
|
}));
|
|
153
141
|
};
|
|
142
|
+
// -- messages ----------------------------------------------------------
|
|
143
|
+
const [loadMessagesViaPluginError, setLoadMessagesViaPluginError] = createSignal();
|
|
144
|
+
const [saveMessagesViaPluginError, setSaveMessagesViaPluginError] = createSignal();
|
|
145
|
+
let messagesQuery;
|
|
146
|
+
let lintReportsQuery;
|
|
147
|
+
let store;
|
|
148
|
+
// wait for seetings to load v2Persistence flag
|
|
149
|
+
// .catch avoids throwing here if the awaitable is rejected
|
|
150
|
+
// error is recorded via markInitAsFailed so no need to capture it again
|
|
151
|
+
await loadedSettings.catch(() => { });
|
|
152
|
+
if (v2Persistence) {
|
|
153
|
+
messagesQuery = stubMessagesQuery;
|
|
154
|
+
lintReportsQuery = stubMessageLintReportsQuery;
|
|
155
|
+
try {
|
|
156
|
+
store = await openStore({ projectPath, nodeishFs, locales });
|
|
157
|
+
markInitAsComplete();
|
|
158
|
+
}
|
|
159
|
+
catch (e) {
|
|
160
|
+
markInitAsFailed(e);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
messagesQuery = createMessagesQuery({
|
|
165
|
+
projectPath,
|
|
166
|
+
nodeishFs,
|
|
167
|
+
settings,
|
|
168
|
+
resolvedModules,
|
|
169
|
+
onInitialMessageLoadResult: (e) => {
|
|
170
|
+
if (e) {
|
|
171
|
+
markInitAsFailed(e);
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
markInitAsComplete();
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
onLoadMessageResult: (e) => {
|
|
178
|
+
setLoadMessagesViaPluginError(e);
|
|
179
|
+
},
|
|
180
|
+
onSaveMessageResult: (e) => {
|
|
181
|
+
setSaveMessagesViaPluginError(e);
|
|
182
|
+
},
|
|
183
|
+
});
|
|
184
|
+
lintReportsQuery = createMessageLintReportsQuery(messagesQuery, settings, installedMessageLintRules, resolvedModules);
|
|
185
|
+
store = undefined;
|
|
186
|
+
}
|
|
154
187
|
// -- app ---------------------------------------------------------------
|
|
155
188
|
const initializeError = await initialized.catch((error) => error);
|
|
156
|
-
const lintReportsQuery = createMessageLintReportsQuery(messagesQuery, settings, installedMessageLintRules, resolvedModules);
|
|
157
189
|
/**
|
|
158
190
|
* Utility to escape reactive tracking and avoid multiple calls to
|
|
159
191
|
* the capture event.
|
|
@@ -179,6 +211,8 @@ export async function loadProject(args) {
|
|
|
179
211
|
settings: settings(),
|
|
180
212
|
installedPluginIds: installedPlugins().map((p) => p.id),
|
|
181
213
|
installedMessageLintRuleIds: installedMessageLintRules().map((r) => r.id),
|
|
214
|
+
// TODO: fix for v2Persistence
|
|
215
|
+
// https://github.com/opral/inlang-message-sdk/issues/78
|
|
182
216
|
numberOfMessages: messagesQuery.includedMessageIds().length,
|
|
183
217
|
},
|
|
184
218
|
});
|
|
@@ -204,6 +238,7 @@ export async function loadProject(args) {
|
|
|
204
238
|
messages: messagesQuery,
|
|
205
239
|
messageLintReports: lintReportsQuery,
|
|
206
240
|
},
|
|
241
|
+
store,
|
|
207
242
|
};
|
|
208
243
|
});
|
|
209
244
|
}
|
package/dist/loadProject.test.js
CHANGED
|
@@ -219,11 +219,15 @@ describe("initialization", () => {
|
|
|
219
219
|
expect(result.error).toBeUndefined();
|
|
220
220
|
expect(result.data).toBeDefined();
|
|
221
221
|
});
|
|
222
|
+
// TODO: fix this test
|
|
223
|
+
// https://github.com/opral/inlang-message-sdk/issues/76
|
|
224
|
+
// it doesn't work because failure to open the settings file doesn't throw
|
|
225
|
+
// errors are returned in project.errors()
|
|
222
226
|
it("should resolve from a windows path", async () => {
|
|
223
227
|
const repo = await mockRepo();
|
|
224
228
|
const fs = repo.nodeishFs;
|
|
225
|
-
fs.mkdir("C:\\Users\\user\\project.inlang", { recursive: true });
|
|
226
|
-
fs.writeFile("C:\\Users\\user\\project.inlang\\settings.json", JSON.stringify(settings));
|
|
229
|
+
await fs.mkdir("C:\\Users\\user\\project.inlang", { recursive: true });
|
|
230
|
+
await fs.writeFile("C:\\Users\\user\\project.inlang\\settings.json", JSON.stringify(settings));
|
|
227
231
|
const result = await tryCatch(() => loadProject({
|
|
228
232
|
projectPath: "C:\\Users\\user\\project.inlang",
|
|
229
233
|
repo,
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* State machine to convert async save() into batched async save()
|
|
3
|
+
* states = idle -> acquiring -> saving -> idle
|
|
4
|
+
* idle = nothing queued, ready to acquire lock.
|
|
5
|
+
* aquiring = waiting to acquire a lock: requests go into the queue.
|
|
6
|
+
* saving = lock is acquired, save has begun: new requests go into the next batch.
|
|
7
|
+
* The next batch should not acquire the lock while current save is in progress.
|
|
8
|
+
* Queued requests are only resolved when the save completes.
|
|
9
|
+
*/
|
|
10
|
+
export declare function batchedIO(acquireLock: () => Promise<number>, releaseLock: (lock: number) => Promise<void>, save: () => Promise<void>): (id: string) => Promise<string>;
|
|
11
|
+
//# sourceMappingURL=batchedIO.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchedIO.d.ts","sourceRoot":"","sources":["../../src/persistence/batchedIO.ts"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CACxB,WAAW,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,EAClC,WAAW,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EAC5C,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACvB,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CA+CjC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import _debug from "debug";
|
|
2
|
+
const debug = _debug("sdk:batchedIO");
|
|
3
|
+
/**
|
|
4
|
+
* State machine to convert async save() into batched async save()
|
|
5
|
+
* states = idle -> acquiring -> saving -> idle
|
|
6
|
+
* idle = nothing queued, ready to acquire lock.
|
|
7
|
+
* aquiring = waiting to acquire a lock: requests go into the queue.
|
|
8
|
+
* saving = lock is acquired, save has begun: new requests go into the next batch.
|
|
9
|
+
* The next batch should not acquire the lock while current save is in progress.
|
|
10
|
+
* Queued requests are only resolved when the save completes.
|
|
11
|
+
*/
|
|
12
|
+
export function batchedIO(acquireLock, releaseLock, save) {
|
|
13
|
+
// 3-state machine
|
|
14
|
+
let state = "idle";
|
|
15
|
+
const queue = [];
|
|
16
|
+
// initialize nextBatch lazily, reset after saving
|
|
17
|
+
let nextBatch = undefined;
|
|
18
|
+
// batched save function
|
|
19
|
+
return async (id) => {
|
|
20
|
+
if (state === "idle") {
|
|
21
|
+
state = "acquiring";
|
|
22
|
+
const lock = await acquireLock();
|
|
23
|
+
state = "saving";
|
|
24
|
+
await save();
|
|
25
|
+
await releaseLock(lock);
|
|
26
|
+
resolveQueued();
|
|
27
|
+
nextBatch = undefined;
|
|
28
|
+
state = "idle";
|
|
29
|
+
return id;
|
|
30
|
+
}
|
|
31
|
+
else if (state === "acquiring") {
|
|
32
|
+
return new Promise((resolve, reject) => {
|
|
33
|
+
queue.push({ id, resolve, reject });
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
else {
|
|
37
|
+
// state === "saving"
|
|
38
|
+
nextBatch = nextBatch ?? batchedIO(acquireLock, releaseLock, save);
|
|
39
|
+
return await nextBatch(id);
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
function resolveQueued() {
|
|
43
|
+
debug("batched", queue.length + 1);
|
|
44
|
+
for (const { id, resolve } of queue) {
|
|
45
|
+
resolve(id);
|
|
46
|
+
}
|
|
47
|
+
queue.length = 0;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batchedIO.test.d.ts","sourceRoot":"","sources":["../../src/persistence/batchedIO.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from "vitest";
|
|
2
|
+
import { sleep } from "../test-utilities/sleep.js";
|
|
3
|
+
import { batchedIO } from "./batchedIO.js";
|
|
4
|
+
let locked = false;
|
|
5
|
+
const instrumentAquireLockStart = vi.fn();
|
|
6
|
+
async function mockAquireLock() {
|
|
7
|
+
instrumentAquireLockStart();
|
|
8
|
+
let pollCount = 0;
|
|
9
|
+
while (locked && pollCount++ < 100) {
|
|
10
|
+
await sleep(10);
|
|
11
|
+
}
|
|
12
|
+
if (locked) {
|
|
13
|
+
throw new Error("Timeout acquiring lock");
|
|
14
|
+
}
|
|
15
|
+
await sleep(10);
|
|
16
|
+
locked = true;
|
|
17
|
+
return 69;
|
|
18
|
+
}
|
|
19
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
20
|
+
async function mockReleaseLock(_) {
|
|
21
|
+
sleep(10);
|
|
22
|
+
locked = false;
|
|
23
|
+
}
|
|
24
|
+
const instrumentSaveStart = vi.fn();
|
|
25
|
+
const instrumentSaveEnd = vi.fn();
|
|
26
|
+
async function mockSave() {
|
|
27
|
+
instrumentSaveStart();
|
|
28
|
+
await sleep(50);
|
|
29
|
+
instrumentSaveEnd();
|
|
30
|
+
}
|
|
31
|
+
describe("batchedIO", () => {
|
|
32
|
+
it("queues 2 requests while waiting for lock and pushes 2 more to the next batch", async () => {
|
|
33
|
+
const save = batchedIO(mockAquireLock, mockReleaseLock, mockSave);
|
|
34
|
+
const p1 = save("1");
|
|
35
|
+
const p2 = save("2");
|
|
36
|
+
await sleep(5);
|
|
37
|
+
expect(instrumentAquireLockStart).toHaveBeenCalledTimes(1);
|
|
38
|
+
expect(instrumentSaveStart).not.toHaveBeenCalled();
|
|
39
|
+
await sleep(10);
|
|
40
|
+
expect(instrumentSaveStart).toHaveBeenCalled();
|
|
41
|
+
expect(instrumentSaveEnd).not.toHaveBeenCalled();
|
|
42
|
+
const p3 = save("3");
|
|
43
|
+
const p4 = save("4");
|
|
44
|
+
expect(instrumentAquireLockStart).toHaveBeenCalledTimes(2);
|
|
45
|
+
expect(locked).toBe(true);
|
|
46
|
+
await sleep(50);
|
|
47
|
+
expect(instrumentSaveEnd).toHaveBeenCalled();
|
|
48
|
+
expect(await p1).toBe("1");
|
|
49
|
+
expect(await p2).toBe("2");
|
|
50
|
+
expect(instrumentSaveStart).toHaveBeenCalledTimes(1);
|
|
51
|
+
expect(await p3).toBe("3");
|
|
52
|
+
expect(await p4).toBe("4");
|
|
53
|
+
expect(instrumentAquireLockStart).toHaveBeenCalledTimes(2);
|
|
54
|
+
expect(instrumentSaveStart).toHaveBeenCalledTimes(2);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"acquireFileLock.d.ts","sourceRoot":"","sources":["../../../src/persistence/filelock/acquireFileLock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAQnD,wBAAsB,eAAe,CACpC,EAAE,EAAE,iBAAiB,EACrB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,GAAE,MAAU,GAClB,OAAO,CAAC,MAAM,CAAC,
|
|
1
|
+
{"version":3,"file":"acquireFileLock.d.ts","sourceRoot":"","sources":["../../../src/persistence/filelock/acquireFileLock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAQnD,wBAAsB,eAAe,CACpC,EAAE,EAAE,iBAAiB,EACrB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,GAAE,MAAU,GAClB,OAAO,CAAC,MAAM,CAAC,CAmHjB"}
|
|
@@ -11,16 +11,18 @@ export async function acquireFileLock(fs, lockDirPath, lockOrigin, tryCount = 0)
|
|
|
11
11
|
try {
|
|
12
12
|
debug(lockOrigin + " tries to acquire a lockfile Retry Nr.: " + tryCount);
|
|
13
13
|
await fs.mkdir(lockDirPath);
|
|
14
|
+
// NOTE: fs.stat does not need to be atomic since mkdir would crash atomically - if we are here its save to consider the lock held by this process
|
|
14
15
|
const stats = await fs.stat(lockDirPath);
|
|
15
16
|
debug(lockOrigin + " acquired a lockfile Retry Nr.: " + tryCount);
|
|
16
17
|
return stats.mtimeMs;
|
|
17
18
|
}
|
|
18
19
|
catch (error) {
|
|
19
20
|
if (error.code !== "EEXIST") {
|
|
20
|
-
// we
|
|
21
|
+
// NOTE in case we have an EEXIST - this is an expected error: the folder existed - another process already acquired the lock. Rethrow all other errors
|
|
21
22
|
throw error;
|
|
22
23
|
}
|
|
23
24
|
}
|
|
25
|
+
// land here if the lockDirPath already exists => lock is held by other process
|
|
24
26
|
let currentLockTime;
|
|
25
27
|
try {
|
|
26
28
|
const stats = await fs.stat(lockDirPath);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"releaseLock.d.ts","sourceRoot":"","sources":["../../../src/persistence/filelock/releaseLock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAInD,wBAAsB,WAAW,CAChC,EAAE,EAAE,iBAAiB,EACrB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"releaseLock.d.ts","sourceRoot":"","sources":["../../../src/persistence/filelock/releaseLock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAInD,wBAAsB,WAAW,CAChC,EAAE,EAAE,iBAAiB,EACrB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,iBAoBhB"}
|
|
@@ -5,8 +5,9 @@ export async function releaseLock(fs, lockDirPath, lockOrigin, lockTime) {
|
|
|
5
5
|
debug(lockOrigin + " releasing the lock ");
|
|
6
6
|
try {
|
|
7
7
|
const stats = await fs.stat(lockDirPath);
|
|
8
|
+
// I believe this check associates the lock with the aquirer
|
|
8
9
|
if (stats.mtimeMs === lockTime) {
|
|
9
|
-
//
|
|
10
|
+
// NOTE: since we have to use a timeout for stale detection (uTimes is not exposed via mermoryfs) the check for the locktime is not sufficient and can fail in rare cases when another process accuires a lock that was identifiert as tale between call to fs.state and rmDir
|
|
10
11
|
await fs.rmdir(lockDirPath);
|
|
11
12
|
}
|
|
12
13
|
}
|