@milaboratories/pl-middle-layer 1.67.5 → 1.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/block_registry/registry-v2-provider.d.ts +1 -3
- package/dist/block_registry/registry-v2-provider.d.ts.map +1 -1
- package/dist/block_registry/registry.d.ts +5 -9
- package/dist/block_registry/registry.d.ts.map +1 -1
- package/dist/block_registry/template_provider.d.ts +3 -5
- package/dist/block_registry/template_provider.d.ts.map +1 -1
- package/dist/block_registry/watcher.d.ts +4 -6
- package/dist/block_registry/watcher.d.ts.map +1 -1
- package/dist/block_registry/well_known_registries.d.ts +3 -5
- package/dist/block_registry/well_known_registries.d.ts.map +1 -1
- package/dist/index.d.ts +6 -2
- package/dist/middle_layer/build_stamp.cjs +34 -0
- package/dist/middle_layer/build_stamp.cjs.map +1 -0
- package/dist/middle_layer/build_stamp.js +34 -0
- package/dist/middle_layer/build_stamp.js.map +1 -0
- package/dist/middle_layer/driver_kit.d.ts +3 -4
- package/dist/middle_layer/driver_kit.d.ts.map +1 -1
- package/dist/middle_layer/index.d.ts +3 -2
- package/dist/middle_layer/middle_layer.cjs +48 -0
- package/dist/middle_layer/middle_layer.cjs.map +1 -1
- package/dist/middle_layer/middle_layer.d.ts +23 -4
- package/dist/middle_layer/middle_layer.d.ts.map +1 -1
- package/dist/middle_layer/middle_layer.js +48 -0
- package/dist/middle_layer/middle_layer.js.map +1 -1
- package/dist/middle_layer/ops.cjs +8 -2
- package/dist/middle_layer/ops.cjs.map +1 -1
- package/dist/middle_layer/ops.d.ts +70 -26
- package/dist/middle_layer/ops.d.ts.map +1 -1
- package/dist/middle_layer/ops.js +8 -2
- package/dist/middle_layer/ops.js.map +1 -1
- package/dist/middle_layer/project.cjs +163 -8
- package/dist/middle_layer/project.cjs.map +1 -1
- package/dist/middle_layer/project.d.ts +56 -7
- package/dist/middle_layer/project.d.ts.map +1 -1
- package/dist/middle_layer/project.js +165 -11
- package/dist/middle_layer/project.js.map +1 -1
- package/dist/middle_layer/project_list.d.ts +5 -3
- package/dist/middle_layer/project_list.d.ts.map +1 -1
- package/dist/middle_layer/sharing_list.d.ts +3 -4
- package/dist/middle_layer/sharing_list.d.ts.map +1 -1
- package/dist/middle_layer/tree_snapshot_store.cjs +283 -0
- package/dist/middle_layer/tree_snapshot_store.cjs.map +1 -0
- package/dist/middle_layer/tree_snapshot_store.d.ts +143 -0
- package/dist/middle_layer/tree_snapshot_store.d.ts.map +1 -0
- package/dist/middle_layer/tree_snapshot_store.js +280 -0
- package/dist/middle_layer/tree_snapshot_store.js.map +1 -0
- package/dist/middle_layer/types.d.ts +1 -3
- package/dist/middle_layer/types.d.ts.map +1 -1
- package/dist/model/block_pack_spec.d.ts +13 -15
- package/dist/model/block_pack_spec.d.ts.map +1 -1
- package/dist/model/frontend.d.ts +1 -2
- package/dist/model/frontend.d.ts.map +1 -1
- package/dist/model/project_helper.d.ts +2 -4
- package/dist/model/project_helper.d.ts.map +1 -1
- package/dist/model/project_model.d.ts +12 -12
- package/dist/model/project_model.d.ts.map +1 -1
- package/dist/model/sharing_model.d.ts +24 -26
- package/dist/model/sharing_model.d.ts.map +1 -1
- package/dist/model/template_apply.d.ts +9 -8
- package/dist/model/template_apply.d.ts.map +1 -1
- package/dist/model/template_export.d.ts +2 -2
- package/dist/model/template_export.d.ts.map +1 -1
- package/dist/model/template_parser.d.ts +2 -4
- package/dist/model/template_parser.d.ts.map +1 -1
- package/dist/model/template_resolve.d.ts +10 -11
- package/dist/model/template_resolve.d.ts.map +1 -1
- package/dist/model/template_serializer.d.ts +6 -5
- package/dist/model/template_serializer.d.ts.map +1 -1
- package/dist/model/template_spec.d.ts +7 -9
- package/dist/model/template_spec.d.ts.map +1 -1
- package/dist/mutator/block-pack/block_pack.d.ts +2 -3
- package/dist/mutator/block-pack/block_pack.d.ts.map +1 -1
- package/dist/mutator/project.cjs +20 -3
- package/dist/mutator/project.cjs.map +1 -1
- package/dist/mutator/project.d.ts +4 -3
- package/dist/mutator/project.d.ts.map +1 -1
- package/dist/mutator/project.js +20 -3
- package/dist/mutator/project.js.map +1 -1
- package/dist/mutator/template/render_template.d.ts +1 -3
- package/dist/mutator/template/render_template.d.ts.map +1 -1
- package/dist/mutator/template/template_cache.d.ts +15 -16
- package/dist/mutator/template/template_cache.d.ts.map +1 -1
- package/dist/mutator/template/template_loading.d.ts +10 -12
- package/dist/mutator/template/template_loading.d.ts.map +1 -1
- package/dist/network_check/network_check.d.ts +3 -5
- package/dist/network_check/network_check.d.ts.map +1 -1
- package/dist/pool/data.d.ts +2 -4
- package/dist/pool/data.d.ts.map +1 -1
- package/dist/pool/driver.d.ts +6 -6
- package/dist/pool/driver.d.ts.map +1 -1
- package/dist/pool/index.d.ts +1 -0
- package/dist/pool/p_object_collection.d.ts +1 -3
- package/dist/pool/p_object_collection.d.ts.map +1 -1
- package/dist/pool/result_pool.d.ts +3 -1
- package/dist/worker/worker.d.ts +1 -1
- package/package.json +14 -14
- package/src/middle_layer/build_stamp.ts +36 -0
- package/src/middle_layer/index.ts +1 -0
- package/src/middle_layer/middle_layer.ts +77 -0
- package/src/middle_layer/ops.ts +46 -1
- package/src/middle_layer/project.ts +234 -14
- package/src/middle_layer/project_failsafe.test.ts +47 -0
- package/src/middle_layer/tree_snapshot_scenarios.test.ts +337 -0
- package/src/middle_layer/tree_snapshot_store.test.ts +331 -0
- package/src/middle_layer/tree_snapshot_store.ts +419 -0
- package/src/mutator/project.ts +27 -5
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { TemplateSpecPrepared } from "../../model/template_spec.js";
|
|
2
2
|
import { BlockPackSpecPrepared } from "../../model/block_pack_spec.js";
|
|
3
|
+
import "../../model/index.js";
|
|
3
4
|
import { AnyResourceRef, PlClient, PlTransaction, ResourceRef, SignedResourceId } from "@milaboratories/pl-client";
|
|
4
5
|
import { CompiledTemplateV3, TemplateData } from "@milaboratories/pl-model-backend";
|
|
5
|
-
|
|
6
6
|
//#region src/mutator/template/template_cache.d.ts
|
|
7
|
-
declare const TemplateCacheType: import("@milaboratories/pl-client").ResourceType;
|
|
8
|
-
declare const TemplateCacheFieldName = "__templateCache";
|
|
7
|
+
export declare const TemplateCacheType: import("@milaboratories/pl-client").ResourceType;
|
|
8
|
+
export declare const TemplateCacheFieldName = "__templateCache";
|
|
9
9
|
/** @internal exported for testing */
|
|
10
|
-
declare const GC_ACCESS_THRESHOLD = 30;
|
|
10
|
+
export declare const GC_ACCESS_THRESHOLD = 30;
|
|
11
11
|
/** @internal exported for testing */
|
|
12
|
-
declare const GC_MAX_ENTRIES = 3000;
|
|
12
|
+
export declare const GC_MAX_ENTRIES = 3000;
|
|
13
13
|
/** @internal exported for testing */
|
|
14
|
-
declare const ACCESS_COUNT_KEY = "_accessCount";
|
|
14
|
+
export declare const ACCESS_COUNT_KEY = "_accessCount";
|
|
15
15
|
/** @internal exported for testing */
|
|
16
|
-
declare const ACCESS_KEY_PREFIX = "access_";
|
|
17
|
-
type TemplateCacheStat = {
|
|
16
|
+
export declare const ACCESS_KEY_PREFIX = "access_";
|
|
17
|
+
export type TemplateCacheStat = {
|
|
18
18
|
totalMs: number;
|
|
19
19
|
flattenMs: number;
|
|
20
20
|
cacheInitMs: number;
|
|
@@ -38,13 +38,13 @@ interface CacheableNode {
|
|
|
38
38
|
childHashes: string[];
|
|
39
39
|
}
|
|
40
40
|
/** Flatten template tree into a topologically ordered list of cacheable nodes (leaves first). */
|
|
41
|
-
declare function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[];
|
|
41
|
+
export declare function flattenTemplateTree(data: TemplateData | CompiledTemplateV3): CacheableNode[];
|
|
42
42
|
/** Clear the in-memory cacheRid entry (call on errors referencing the cache resource). */
|
|
43
|
-
declare function invalidateTemplateCacheId(pl: PlClient): void;
|
|
43
|
+
export declare function invalidateTemplateCacheId(pl: PlClient): void;
|
|
44
44
|
/** Find or create the TemplateCache/1 resource on user root. */
|
|
45
|
-
declare function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId>;
|
|
45
|
+
export declare function getOrCreateTemplateCache(pl: PlClient): Promise<SignedResourceId>;
|
|
46
46
|
/** Remove the template cache from user root. */
|
|
47
|
-
declare function dropTemplateCache(pl: PlClient): Promise<void>;
|
|
47
|
+
export declare function dropTemplateCache(pl: PlClient): Promise<void>;
|
|
48
48
|
/**
|
|
49
49
|
* Run count-based garbage collection on the template cache.
|
|
50
50
|
* Evicts least-recently-used entries when the cache exceeds maxEntries.
|
|
@@ -53,14 +53,14 @@ declare function dropTemplateCache(pl: PlClient): Promise<void>;
|
|
|
53
53
|
* @internal exported for testing (maxEntries parameter allows low thresholds in tests)
|
|
54
54
|
* @returns true if entries were evicted
|
|
55
55
|
*/
|
|
56
|
-
declare function runGc(pl: PlClient, cacheRid: SignedResourceId, maxEntries?: number): Promise<boolean>;
|
|
56
|
+
export declare function runGc(pl: PlClient, cacheRid: SignedResourceId, maxEntries?: number): Promise<boolean>;
|
|
57
57
|
/**
|
|
58
58
|
* Materialize a template tree via the cache.
|
|
59
59
|
* Manages its own transactions internally — do NOT call inside an existing tx.
|
|
60
60
|
*
|
|
61
61
|
* @returns concrete SignedResourceId of the root template
|
|
62
62
|
*/
|
|
63
|
-
declare function loadTemplateCached(pl: PlClient, spec: TemplateSpecPrepared, options?: {
|
|
63
|
+
export declare function loadTemplateCached(pl: PlClient, spec: TemplateSpecPrepared, options?: {
|
|
64
64
|
cacheResourceId?: SignedResourceId;
|
|
65
65
|
}): Promise<SignedResourceId>;
|
|
66
66
|
/**
|
|
@@ -68,9 +68,8 @@ declare function loadTemplateCached(pl: PlClient, spec: TemplateSpecPrepared, op
|
|
|
68
68
|
* Returns a copy of the spec with the template replaced by a cached reference.
|
|
69
69
|
* If the template is already cached, returns the spec unchanged.
|
|
70
70
|
*/
|
|
71
|
-
declare function cacheBlockPackTemplate(pl: PlClient, spec: BlockPackSpecPrepared, options?: {
|
|
71
|
+
export declare function cacheBlockPackTemplate(pl: PlClient, spec: BlockPackSpecPrepared, options?: {
|
|
72
72
|
cacheResourceId?: SignedResourceId;
|
|
73
73
|
}): Promise<BlockPackSpecPrepared>;
|
|
74
74
|
//#endregion
|
|
75
|
-
export { ACCESS_COUNT_KEY, ACCESS_KEY_PREFIX, GC_ACCESS_THRESHOLD, GC_MAX_ENTRIES, TemplateCacheFieldName, TemplateCacheStat, TemplateCacheType, cacheBlockPackTemplate, dropTemplateCache, flattenTemplateTree, getOrCreateTemplateCache, invalidateTemplateCacheId, loadTemplateCached, runGc };
|
|
76
75
|
//# sourceMappingURL=template_cache.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template_cache.d.ts","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"template_cache.d.ts","names":[],"sources":["../../../src/mutator/template/template_cache.ts"],"mappings":";;;;;;qBAqCa,uDAAiB;qBAEjB;;qBAGA;;qBAEA;;qBAEA;;qBAEA;YAED;EACV;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAoBQ;;EAER;;;EAGA,SAAS,IAAI,eAAe,WAAW,oBAAoB,oBAAoB;;EAE/E;;;wBA2Vc,oBAAoB,MAAM,eAAe,qBAAqB;;wBAY9D,0BAA0B,IAAI;;wBAKxB,yBAAyB,IAAI,WAAW,QAAQ;;wBA+BhD,kBAAkB,IAAI,WAAW;;;;;;;;;wBAoBjC,MACpB,IAAI,UACJ,UAAU,kBACV,sBACC;;;;;;;wBA8LmB,mBACpB,IAAI,UACJ,MAAM,sBACN;EAAY,kBAAkB;IAC7B,QAAQ;;;;;;wBAiFW,uBACpB,IAAI,UACJ,MAAM,uBACN;EAAY,kBAAkB;IAC7B,QAAQ"}
|
|
@@ -1,17 +1,15 @@
|
|
|
1
1
|
import { TemplateSpecAny, TemplateSpecPrepared } from "../../model/template_spec.js";
|
|
2
2
|
import { AnyRef, PlTransaction, ResourceType } from "@milaboratories/pl-client";
|
|
3
|
-
|
|
4
3
|
//#region src/mutator/template/template_loading.d.ts
|
|
5
|
-
declare const TengoTemplateGet: ResourceType;
|
|
6
|
-
declare const TengoTemplateGetRegistry = "registry";
|
|
7
|
-
declare const TengoTemplateGetTemplateURI = "templateURI";
|
|
8
|
-
declare const TengoTemplateGetTemplate = "template";
|
|
9
|
-
declare const TengoTemplatePack: ResourceType;
|
|
10
|
-
declare const TengoTemplatePackConvert: ResourceType;
|
|
11
|
-
declare const TengoTemplatePackConvertTemplatePack = "templatePack";
|
|
12
|
-
declare const TengoTemplatePackConvertTemplate = "template";
|
|
13
|
-
declare function prepareTemplateSpec(tpl: TemplateSpecAny): Promise<TemplateSpecPrepared>;
|
|
14
|
-
declare function loadTemplate(tx: PlTransaction, spec: TemplateSpecPrepared): AnyRef;
|
|
4
|
+
export declare const TengoTemplateGet: ResourceType;
|
|
5
|
+
export declare const TengoTemplateGetRegistry = "registry";
|
|
6
|
+
export declare const TengoTemplateGetTemplateURI = "templateURI";
|
|
7
|
+
export declare const TengoTemplateGetTemplate = "template";
|
|
8
|
+
export declare const TengoTemplatePack: ResourceType;
|
|
9
|
+
export declare const TengoTemplatePackConvert: ResourceType;
|
|
10
|
+
export declare const TengoTemplatePackConvertTemplatePack = "templatePack";
|
|
11
|
+
export declare const TengoTemplatePackConvertTemplate = "template";
|
|
12
|
+
export declare function prepareTemplateSpec(tpl: TemplateSpecAny): Promise<TemplateSpecPrepared>;
|
|
13
|
+
export declare function loadTemplate(tx: PlTransaction, spec: TemplateSpecPrepared): AnyRef;
|
|
15
14
|
//#endregion
|
|
16
|
-
export { TengoTemplateGet, TengoTemplateGetRegistry, TengoTemplateGetTemplate, TengoTemplateGetTemplateURI, TengoTemplatePack, TengoTemplatePackConvert, TengoTemplatePackConvertTemplate, TengoTemplatePackConvertTemplatePack, loadTemplate, prepareTemplateSpec };
|
|
17
15
|
//# sourceMappingURL=template_loading.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"template_loading.d.ts","names":[],"sources":["../../../src/mutator/template/template_loading.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"template_loading.d.ts","names":[],"sources":["../../../src/mutator/template/template_loading.ts"],"mappings":";;;qBAea,kBAAkB;qBAClB;qBACA;qBACA;qBAEA,mBAAmB;qBACnB,0BAA0B;qBAI1B;qBACA;wBAES,oBAAoB,KAAK,kBAAkB,QAAQ;wBAgCzD,aAAa,IAAI,eAAe,MAAM,uBAAuB"}
|
|
@@ -2,9 +2,8 @@ import { Dispatcher } from "undici";
|
|
|
2
2
|
import { MiLogger, Signer } from "@milaboratories/ts-helpers";
|
|
3
3
|
import { PlClient, PlClientConfig } from "@milaboratories/pl-client";
|
|
4
4
|
import { ClientDownload, ClientUpload, LsDriver } from "@milaboratories/pl-drivers";
|
|
5
|
-
|
|
6
5
|
//#region src/network_check/network_check.d.ts
|
|
7
|
-
interface CheckNetworkOpts {
|
|
6
|
+
export interface CheckNetworkOpts {
|
|
8
7
|
/** Signal to abort all network checks. */
|
|
9
8
|
signal?: AbortSignal;
|
|
10
9
|
/** Platforma Backend pings options. */
|
|
@@ -40,8 +39,8 @@ interface CheckNetworkOpts {
|
|
|
40
39
|
/** Checks connectivity to Platforma Backend, to block registry
|
|
41
40
|
* and to auto-update CDN,
|
|
42
41
|
* and generates a string report. */
|
|
43
|
-
declare function checkNetwork(plCredentials: string, plUser: string | undefined, plPassword: string | undefined, optsOverrides?: Partial<CheckNetworkOpts>): Promise<string>;
|
|
44
|
-
declare function initNetworkCheck(plCredentials: string, plUser: string | undefined, plPassword: string | undefined, optsOverrides?: Partial<CheckNetworkOpts>): Promise<{
|
|
42
|
+
export declare function checkNetwork(plCredentials: string, plUser: string | undefined, plPassword: string | undefined, optsOverrides?: Partial<CheckNetworkOpts>): Promise<string>;
|
|
43
|
+
export declare function initNetworkCheck(plCredentials: string, plUser: string | undefined, plPassword: string | undefined, optsOverrides?: Partial<CheckNetworkOpts>): Promise<{
|
|
45
44
|
logger: MiLogger;
|
|
46
45
|
plConfig: PlClientConfig;
|
|
47
46
|
signer: Signer;
|
|
@@ -54,5 +53,4 @@ declare function initNetworkCheck(plCredentials: string, plUser: string | undefi
|
|
|
54
53
|
terminate: () => Promise<void>;
|
|
55
54
|
}>;
|
|
56
55
|
//#endregion
|
|
57
|
-
export { CheckNetworkOpts, checkNetwork, initNetworkCheck };
|
|
58
56
|
//# sourceMappingURL=network_check.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"network_check.d.ts","names":[],"sources":["../../src/network_check/network_check.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"network_check.d.ts","names":[],"sources":["../../src/network_check/network_check.ts"],"mappings":";;;;;iBAqEiB;;EAEf,SAAS;;EAGT;EACA;EACA;;EAGA;;EAGA;EACA;EACA;EACA;EACA;EACA;;EAGA;EACA;EACA;;EAGA;;EAGA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;wBAMoB,aACpB,uBACA,4BACA,gCACA,gBAAe,QAAQ,oBACtB;wBA2FmB,iBACpB,uBACA,4BACA,gCACA,gBAAe,QAAQ,oBACtB;EACD,QAAQ;EACR,UAAU;EACV,QAAQ;EACR,QAAQ;EACR,gBAAgB;EAChB,kBAAkB;EAClB,UAAU;EACV,YAAY;EACZ,KAAK;EACL,iBAAiB"}
|
package/dist/pool/data.d.ts
CHANGED
|
@@ -3,10 +3,8 @@ import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
|
3
3
|
import { ResourceType, SignedResourceId } from "@milaboratories/pl-client";
|
|
4
4
|
import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
|
|
5
5
|
import { OnDemandBlobResourceSnapshot } from "@milaboratories/pl-drivers";
|
|
6
|
-
|
|
7
6
|
//#region src/pool/data.d.ts
|
|
8
|
-
declare function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId;
|
|
9
|
-
declare function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId;
|
|
7
|
+
export declare function deriveGlobalPObjectId(blockId: string, exportName: string): PObjectId;
|
|
8
|
+
export declare function deriveLocalPObjectId(resolvePath: string[], outputName: string): PObjectId;
|
|
10
9
|
//#endregion
|
|
11
|
-
export { deriveGlobalPObjectId, deriveLocalPObjectId };
|
|
12
10
|
//# sourceMappingURL=data.d.ts.map
|
package/dist/pool/data.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"data.d.ts","names":[],"sources":["../../src/pool/data.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"data.d.ts","names":[],"sources":["../../src/pool/data.ts"],"mappings":";;;;;;wBA6UgB,sBAAsB,iBAAiB,qBAAqB;wBAI5D,qBAAqB,uBAAuB,qBAAqB"}
|
package/dist/pool/driver.d.ts
CHANGED
|
@@ -2,19 +2,19 @@ import { PColumnDataUniversal } from "@platforma-sdk/model";
|
|
|
2
2
|
import { PFrameInternal } from "@milaboratories/pl-model-middle-layer";
|
|
3
3
|
import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
|
|
4
4
|
import { DownloadDriver } from "@milaboratories/pl-drivers";
|
|
5
|
+
import "@milaboratories/helpers";
|
|
5
6
|
import { AbstractInternalPFrameDriver, AbstractPFrameDriverOps } from "@milaboratories/pf-driver";
|
|
6
|
-
|
|
7
7
|
//#region src/pool/driver.d.ts
|
|
8
|
-
interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {}
|
|
9
|
-
type PFrameDriverOps = AbstractPFrameDriverOps & {
|
|
10
|
-
/** Port to run parquet HTTP server on. */
|
|
8
|
+
export interface InternalPFrameDriver extends AbstractInternalPFrameDriver<PColumnDataUniversal<PlTreeNodeAccessor>> {}
|
|
9
|
+
export type PFrameDriverOps = AbstractPFrameDriverOps & {
|
|
10
|
+
/** Port to run parquet HTTP server on. */
|
|
11
|
+
parquetServerPort: number;
|
|
11
12
|
};
|
|
12
13
|
/**
|
|
13
14
|
* Tuning for the persistent parquet blob cache, minus its filesystem path (which
|
|
14
15
|
* is supplied separately - see {@link createPFrameDriver}'s `cachePath`). Carries
|
|
15
16
|
* only the size/eviction knobs of {@link PFrameInternal.CacheConfig}.
|
|
16
17
|
*/
|
|
17
|
-
type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, "cachePath">;
|
|
18
|
+
export type ParquetCacheOps = Omit<PFrameInternal.CacheConfig, "cachePath">;
|
|
18
19
|
//#endregion
|
|
19
|
-
export { InternalPFrameDriver, PFrameDriverOps, ParquetCacheOps };
|
|
20
20
|
//# sourceMappingURL=driver.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;;
|
|
1
|
+
{"version":3,"file":"driver.d.ts","names":[],"sources":["../../src/pool/driver.ts"],"mappings":";;;;;;;iBA4SiB,6BAA6B,6BAC5C,qBAAqB;YAGX,kBAAkB;;EAE5B;;;;;;;YAaU,kBAAkB,KAAK,eAAe"}
|
package/dist/pool/index.d.ts
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import { PObject } from "@platforma-sdk/model";
|
|
2
2
|
import { PlTreeNodeAccessor } from "@milaboratories/pl-tree";
|
|
3
|
-
|
|
4
3
|
//#region src/pool/p_object_collection.d.ts
|
|
5
|
-
declare function parseFinalPObjectCollection(node: PlTreeNodeAccessor, errorOnUnknownField: boolean | undefined, prefix: string | undefined, resolvePath: string[]): Record<string, PObject<PlTreeNodeAccessor>>;
|
|
4
|
+
export declare function parseFinalPObjectCollection(node: PlTreeNodeAccessor, errorOnUnknownField: boolean | undefined, prefix: string | undefined, resolvePath: string[]): Record<string, PObject<PlTreeNodeAccessor>>;
|
|
6
5
|
//#endregion
|
|
7
|
-
export { parseFinalPObjectCollection };
|
|
8
6
|
//# sourceMappingURL=p_object_collection.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"p_object_collection.d.ts","names":[],"sources":["../../src/pool/p_object_collection.ts"],"mappings":"
|
|
1
|
+
{"version":3,"file":"p_object_collection.d.ts","names":[],"sources":["../../src/pool/p_object_collection.ts"],"mappings":";;;wBA+FgB,4BACd,MAAM,oBACN,0CACA,4BACA,wBACC,eAAe,QAAQ"}
|
package/dist/worker/worker.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export {
|
|
1
|
+
export {}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-middle-layer",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.68.0",
|
|
4
4
|
"description": "Pl Middle Layer",
|
|
5
5
|
"keywords": [],
|
|
6
6
|
"license": "UNLICENSED",
|
|
@@ -31,33 +31,33 @@
|
|
|
31
31
|
"yaml": "^2.8.0",
|
|
32
32
|
"zod": "~3.25.76",
|
|
33
33
|
"@milaboratories/columns-collection-driver": "0.2.4",
|
|
34
|
-
"@milaboratories/computable": "2.9.8",
|
|
35
|
-
"@milaboratories/pl-deployments": "3.0.16",
|
|
36
|
-
"@milaboratories/pf-spec-driver": "1.5.1",
|
|
37
34
|
"@milaboratories/helpers": "1.14.5",
|
|
38
|
-
"@milaboratories/pl-client": "3.14.7",
|
|
39
35
|
"@milaboratories/pf-driver": "1.9.1",
|
|
40
|
-
"@milaboratories/
|
|
36
|
+
"@milaboratories/pf-spec-driver": "1.5.1",
|
|
37
|
+
"@milaboratories/computable": "2.9.8",
|
|
38
|
+
"@milaboratories/pl-drivers": "1.16.17",
|
|
39
|
+
"@milaboratories/pl-client": "3.14.7",
|
|
41
40
|
"@milaboratories/pl-errors": "1.4.36",
|
|
41
|
+
"@milaboratories/pl-http": "1.2.4",
|
|
42
42
|
"@milaboratories/pl-model-backend": "1.4.21",
|
|
43
|
+
"@milaboratories/pl-deployments": "3.0.16",
|
|
43
44
|
"@milaboratories/pl-model-common": "1.48.0",
|
|
45
|
+
"@milaboratories/pl-tree": "1.14.0",
|
|
44
46
|
"@milaboratories/pl-model-middle-layer": "1.32.0",
|
|
45
|
-
"@milaboratories/pl-http": "1.2.4",
|
|
46
|
-
"@milaboratories/resolve-helper": "1.1.3",
|
|
47
|
-
"@milaboratories/pl-tree": "1.13.7",
|
|
48
47
|
"@platforma-sdk/block-tools": "2.14.3",
|
|
49
|
-
"@
|
|
48
|
+
"@milaboratories/resolve-helper": "1.1.3",
|
|
49
|
+
"@milaboratories/ts-helpers": "1.8.6",
|
|
50
50
|
"@platforma-sdk/model": "1.83.0",
|
|
51
|
-
"@
|
|
51
|
+
"@platforma-sdk/workflow-tengo": "6.8.3"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@types/node": "~24.5.2",
|
|
55
55
|
"@types/semver": "^7.7.0",
|
|
56
56
|
"@vitest/coverage-istanbul": "^4.1.3",
|
|
57
|
-
"typescript": "
|
|
57
|
+
"typescript": "7.0.2",
|
|
58
58
|
"vitest": "^4.1.3",
|
|
59
|
-
"@milaboratories/build-configs": "2.0.
|
|
60
|
-
"@milaboratories/ts-builder": "1.7.
|
|
59
|
+
"@milaboratories/build-configs": "2.0.1",
|
|
60
|
+
"@milaboratories/ts-builder": "1.7.1",
|
|
61
61
|
"@milaboratories/ts-configs": "1.4.0"
|
|
62
62
|
},
|
|
63
63
|
"engines": {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/** Injected by rolldown at build time, see `build.node.config.js`. Absent when the package is
|
|
2
|
+
* consumed straight from sources (`USE_SOURCES=1`), because no build step runs then. */
|
|
3
|
+
declare const __PL_ML_BUILD_STAMP__: string | undefined;
|
|
4
|
+
|
|
5
|
+
function injectedStamp(): string | undefined {
|
|
6
|
+
try {
|
|
7
|
+
// Read inside a try: with no build step the identifier is an undeclared global, and
|
|
8
|
+
// reading it throws a ReferenceError rather than yielding undefined.
|
|
9
|
+
return __PL_ML_BUILD_STAMP__;
|
|
10
|
+
} catch {
|
|
11
|
+
return undefined;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Identifies the build of this package, and through it the rules that shape what a persisted
|
|
17
|
+
* tree mirror contains: the pruning function, the field filter and the traversal stop rules,
|
|
18
|
+
* all of which live in this package. (The finality predicate comes from pl-client and is NOT
|
|
19
|
+
* covered, which is harmless: finality is recomputed on restore, so it is the one rule that
|
|
20
|
+
* cannot poison a stored file.)
|
|
21
|
+
*
|
|
22
|
+
* Used as a cache-key component, so a change to those rules invalidates every snapshot, costing
|
|
23
|
+
* one cold open. Each built artifact bakes in one stamp, so reopens stay warm across restarts
|
|
24
|
+
* of an installed version. A build from a dirty worktree includes the build time, so editing
|
|
25
|
+
* those rules locally can never hit a snapshot written under the old ones. In practice release
|
|
26
|
+
* builds are dirty too, because CI writes version bumps into the worktree before building; that
|
|
27
|
+
* costs nothing, since the stamp only has to be stable within an artifact.
|
|
28
|
+
*
|
|
29
|
+
* With no build at all (sources mode) the value is a constant. That deliberately trades away
|
|
30
|
+
* the dirty-worktree guarantee: it means someone running from sources exercises the restore
|
|
31
|
+
* path at all, rather than every snapshot being a guaranteed miss for the one audience most
|
|
32
|
+
* likely to find its defects. The exposure it reintroduces, editing pruning rules from sources
|
|
33
|
+
* and hitting a mirror written under the old ones, is the local-development gap the design
|
|
34
|
+
* already accepts, and `treeSnapshots: false` or deleting the directory clears it.
|
|
35
|
+
*/
|
|
36
|
+
export const ML_BUILD_STAMP: string = injectedStamp() ?? "sources";
|
|
@@ -2,5 +2,6 @@ export { MiddleLayer } from "./middle_layer";
|
|
|
2
2
|
export { Project } from "./project";
|
|
3
3
|
export * from "./driver_kit";
|
|
4
4
|
export * from "./ops";
|
|
5
|
+
export type { TreeSnapshotMiss, TreeSnapshotStat } from "./tree_snapshot_store";
|
|
5
6
|
export { ProjectsField } from "./project_list";
|
|
6
7
|
export type { OutgoingShare, PendingShare } from "./sharing_list";
|
|
@@ -111,6 +111,13 @@ import type { Dispatcher } from "undici";
|
|
|
111
111
|
import { RetryAgent } from "undici";
|
|
112
112
|
import { getDebugFlags } from "../debug";
|
|
113
113
|
import { ProjectHelper } from "../model/project_helper";
|
|
114
|
+
import type { TreeSnapshotStat } from "./tree_snapshot_store";
|
|
115
|
+
import { TreeSnapshotStore } from "./tree_snapshot_store";
|
|
116
|
+
|
|
117
|
+
/** How long shutdown waits for close-boundary snapshot writes that are already running. Long
|
|
118
|
+
* enough for a ten-megabyte encode and write on ordinary storage, short enough that a wedged
|
|
119
|
+
* filesystem does not hold the quit open. */
|
|
120
|
+
const SNAPSHOT_DRAIN_TIMEOUT_MS = 5_000;
|
|
114
121
|
|
|
115
122
|
export interface MiddleLayerEnvironment {
|
|
116
123
|
dispose(): Promise<void>;
|
|
@@ -129,6 +136,9 @@ export interface MiddleLayerEnvironment {
|
|
|
129
136
|
readonly driverKit: MiddleLayerDriverKit;
|
|
130
137
|
readonly serviceRegistry: ModelServiceRegistry;
|
|
131
138
|
readonly projectHelper: ProjectHelper;
|
|
139
|
+
/** Persisted project tree mirrors. Undefined when snapshots are switched off, or when the
|
|
140
|
+
* client is impersonating another user, in which case nothing is read or written. */
|
|
141
|
+
readonly treeSnapshots?: TreeSnapshotStore;
|
|
132
142
|
}
|
|
133
143
|
|
|
134
144
|
/**
|
|
@@ -1132,6 +1142,36 @@ export class MiddleLayer {
|
|
|
1132
1142
|
|
|
1133
1143
|
private readonly openedProjects = new Map<ProjectId, Project>();
|
|
1134
1144
|
|
|
1145
|
+
/** Snapshot writes started by {@link closeProject} and not yet finished. Held only so
|
|
1146
|
+
* {@link close} can give them a bounded chance to land. */
|
|
1147
|
+
private readonly pendingSnapshotWrites = new Set<Promise<void>>();
|
|
1148
|
+
|
|
1149
|
+
private trackSnapshotWrite(write: Promise<void>): void {
|
|
1150
|
+
this.pendingSnapshotWrites.add(write);
|
|
1151
|
+
void write.finally(() => this.pendingSnapshotWrites.delete(write));
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/** Waits for close-boundary snapshot writes that are already running, up to `timeoutMs`.
|
|
1155
|
+
*
|
|
1156
|
+
* This starts no work: quitting still performs no snapshot of its own. It only lets a write
|
|
1157
|
+
* that a project close already began finish, so closing a project and immediately quitting
|
|
1158
|
+
* does not routinely lose it. Bounded, because a wedged filesystem must not hang the quit,
|
|
1159
|
+
* and losing the write costs one cold open rather than any correctness. */
|
|
1160
|
+
private async drainSnapshotWrites(timeoutMs: number): Promise<void> {
|
|
1161
|
+
if (this.pendingSnapshotWrites.size === 0) return;
|
|
1162
|
+
|
|
1163
|
+
let timer: NodeJS.Timeout | undefined;
|
|
1164
|
+
const expiry = new Promise<void>((resolve) => {
|
|
1165
|
+
timer = setTimeout(resolve, timeoutMs);
|
|
1166
|
+
timer.unref?.();
|
|
1167
|
+
});
|
|
1168
|
+
try {
|
|
1169
|
+
await Promise.race([Promise.allSettled(this.pendingSnapshotWrites), expiry]);
|
|
1170
|
+
} finally {
|
|
1171
|
+
if (timer !== undefined) clearTimeout(timer);
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1135
1175
|
/** Opens a project, and starts corresponding project maintenance loop. */
|
|
1136
1176
|
public async openProject(id: ProjectId): Promise<void> {
|
|
1137
1177
|
if (this.openedProjects.has(id)) throw new Error(`Project ${id} already opened`);
|
|
@@ -1145,6 +1185,16 @@ export class MiddleLayer {
|
|
|
1145
1185
|
const prj = this.openedProjects.get(id);
|
|
1146
1186
|
if (prj === undefined) throw new Error(`Project ${id} not found among opened projects`);
|
|
1147
1187
|
this.openedProjects.delete(id);
|
|
1188
|
+
|
|
1189
|
+
// Snapshot before destroy, and here rather than inside destroy(): destroy() is also what
|
|
1190
|
+
// application shutdown runs, and quitting should perform no snapshot work. Terminating the
|
|
1191
|
+
// tree invalidates it, so the state has to be taken first either way.
|
|
1192
|
+
//
|
|
1193
|
+
// Started, not awaited. The capture happens synchronously inside, which is the part that
|
|
1194
|
+
// needs the tree alive; the encode and write are up to ten megabytes of work that closing a
|
|
1195
|
+
// project should not sit behind. Kept so shutdown can drain it.
|
|
1196
|
+
this.trackSnapshotWrite(prj.snapshotOnClose());
|
|
1197
|
+
|
|
1148
1198
|
await prj.destroy();
|
|
1149
1199
|
this.openedProjectsList.setValue([...this.openedProjects.keys()]);
|
|
1150
1200
|
}
|
|
@@ -1161,6 +1211,13 @@ export class MiddleLayer {
|
|
|
1161
1211
|
return this.openedProjects.has(id);
|
|
1162
1212
|
}
|
|
1163
1213
|
|
|
1214
|
+
/** Counters for the persisted project tree mirrors, or undefined when they are switched off.
|
|
1215
|
+
* Reads and hits are what show whether a reopen was actually warm, and the miss breakdown
|
|
1216
|
+
* says why it was not. */
|
|
1217
|
+
public get treeSnapshotStats(): Readonly<TreeSnapshotStat> | undefined {
|
|
1218
|
+
return this.env.treeSnapshots?.getStats();
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1164
1221
|
/**
|
|
1165
1222
|
* Deallocates all runtime resources consumed by this object and awaits
|
|
1166
1223
|
* actual termination of event loops and other processes associated with
|
|
@@ -1176,6 +1233,7 @@ export class MiddleLayer {
|
|
|
1176
1233
|
this.sharingStateTree.terminate(),
|
|
1177
1234
|
this.pendingSharesTree.terminate(),
|
|
1178
1235
|
]);
|
|
1236
|
+
await this.drainSnapshotWrites(SNAPSHOT_DRAIN_TIMEOUT_MS);
|
|
1179
1237
|
await this.env.dispose();
|
|
1180
1238
|
await this.pl.close();
|
|
1181
1239
|
}
|
|
@@ -1283,6 +1341,24 @@ export class MiddleLayer {
|
|
|
1283
1341
|
|
|
1284
1342
|
const serviceRegistry = createModelServiceRegistry({ logger });
|
|
1285
1343
|
|
|
1344
|
+
const treeSnapshots = TreeSnapshotStore.create(pl, {
|
|
1345
|
+
dir: ops.treeSnapshotPath,
|
|
1346
|
+
maxSizeBytes: ops.treeSnapshotOps.maxSizeBytes,
|
|
1347
|
+
enabled: ops.treeSnapshotOps.enabled,
|
|
1348
|
+
logger,
|
|
1349
|
+
});
|
|
1350
|
+
if (ops.treeSnapshotOps.enabled) {
|
|
1351
|
+
// Housekeeping before any project opens: drop snapshots from other builds, backends and
|
|
1352
|
+
// users, then trim to the ceiling.
|
|
1353
|
+
await treeSnapshots?.evict();
|
|
1354
|
+
} else {
|
|
1355
|
+
// Switched off, so reclaim what earlier sessions left on disk. The reason to reach for
|
|
1356
|
+
// this switch is usually the disk itself, and leaving the files behind would answer the
|
|
1357
|
+
// wrong half of that complaint. Keyed on the setting, not on the store being absent: it
|
|
1358
|
+
// is also absent for an impersonated client, whose session must not delete anything.
|
|
1359
|
+
await TreeSnapshotStore.purge(ops.treeSnapshotPath, logger);
|
|
1360
|
+
}
|
|
1361
|
+
|
|
1286
1362
|
const env: MiddleLayerEnvironment = {
|
|
1287
1363
|
pl,
|
|
1288
1364
|
blockEventDispatcher: new BlockEventDispatcher(),
|
|
@@ -1303,6 +1379,7 @@ export class MiddleLayer {
|
|
|
1303
1379
|
serviceRegistry,
|
|
1304
1380
|
quickJs,
|
|
1305
1381
|
projectHelper: new ProjectHelper(quickJs, logger),
|
|
1382
|
+
treeSnapshots,
|
|
1306
1383
|
dispose: async () => {
|
|
1307
1384
|
await serviceRegistry.dispose();
|
|
1308
1385
|
await retryHttpDispatcher.destroy();
|
package/src/middle_layer/ops.ts
CHANGED
|
@@ -222,6 +222,40 @@ export type DriverKitOpsConstructor = Omit<
|
|
|
222
222
|
export type MiddleLayerOpsPaths = DriverKitOpsPaths & {
|
|
223
223
|
/** Common root where to put frontend code. */
|
|
224
224
|
readonly frontendDownloadPath: string;
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Directory holding persisted project tree mirrors, one file per project. Like
|
|
228
|
+
* {@link DriverKitOpsPaths.parquetCachePath} and unlike the spill directories, it is NOT
|
|
229
|
+
* emptied on startup: surviving a restart is the entire point. It is pruned instead, see
|
|
230
|
+
* {@link TreeSnapshotOps.maxSizeBytes}.
|
|
231
|
+
*/
|
|
232
|
+
readonly treeSnapshotPath: string;
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
/** Tuning for the persisted project tree mirrors. Their directory is
|
|
236
|
+
* {@link MiddleLayerOpsPaths.treeSnapshotPath}; this carries the behaviour knobs. */
|
|
237
|
+
export type TreeSnapshotOps = {
|
|
238
|
+
/**
|
|
239
|
+
* Whether project tree mirrors are persisted and restored at all.
|
|
240
|
+
*
|
|
241
|
+
* On by default. This is an operational kill switch, for a deployment where the cache
|
|
242
|
+
* directory turns out to be unwritable or otherwise troublesome, not a rollout gate: the
|
|
243
|
+
* floor of the feature is current behaviour, since a cache that never hits is a cold open.
|
|
244
|
+
*/
|
|
245
|
+
readonly enabled: boolean;
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Minimum wall-clock gap between periodic writes for one project.
|
|
249
|
+
*
|
|
250
|
+
* Can be generous, because a stale snapshot is less complete rather than wrong: final
|
|
251
|
+
* resources never change and are never refetched, so this only bounds how much recent work
|
|
252
|
+
* comes back from the non-final frontier on restore.
|
|
253
|
+
*/
|
|
254
|
+
readonly writeInterval: number;
|
|
255
|
+
|
|
256
|
+
/** Total bytes the snapshot directory may occupy after startup eviction. Needed because a
|
|
257
|
+
* heavy project runs to roughly ten megabytes. */
|
|
258
|
+
readonly maxSizeBytes: number;
|
|
225
259
|
};
|
|
226
260
|
|
|
227
261
|
/** Debug options for middle layer. */
|
|
@@ -253,6 +287,10 @@ export type MiddleLayerOpsSettings = DriverKitOpsSettings & {
|
|
|
253
287
|
* `sharedAt + envelopeTtlMs`. Share-with-everybody envelopes never expire
|
|
254
288
|
* (`expiresAt: null`) and ignore this. */
|
|
255
289
|
readonly envelopeTtlMs: number;
|
|
290
|
+
|
|
291
|
+
/** Settings for persisting project tree mirrors to disk, so reopening a project transfers
|
|
292
|
+
* what changed rather than the tree again. */
|
|
293
|
+
readonly treeSnapshotOps: TreeSnapshotOps;
|
|
256
294
|
};
|
|
257
295
|
|
|
258
296
|
export type MiddleLayerOps = MiddleLayerOpsSettings & MiddleLayerOpsPaths;
|
|
@@ -266,6 +304,7 @@ export const DefaultMiddleLayerOpsSettings: Pick<
|
|
|
266
304
|
| "devBlockUpdateRecheckInterval"
|
|
267
305
|
| "debugOps"
|
|
268
306
|
| "envelopeTtlMs"
|
|
307
|
+
| "treeSnapshotOps"
|
|
269
308
|
> = {
|
|
270
309
|
...DefaultDriverKitOpsSettings,
|
|
271
310
|
defaultTreeOptions: {
|
|
@@ -279,17 +318,23 @@ export const DefaultMiddleLayerOpsSettings: Pick<
|
|
|
279
318
|
devBlockUpdateRecheckInterval: 1000,
|
|
280
319
|
projectRefreshInterval: 2000,
|
|
281
320
|
envelopeTtlMs: 14 * 24 * 3600 * 1000, // 14 days
|
|
321
|
+
treeSnapshotOps: {
|
|
322
|
+
enabled: true,
|
|
323
|
+
writeInterval: 5 * 60 * 1000, // 5 minutes
|
|
324
|
+
maxSizeBytes: 256 * 1024 * 1024, // 256 MB, roughly 25 heavy projects
|
|
325
|
+
},
|
|
282
326
|
};
|
|
283
327
|
|
|
284
328
|
export function DefaultMiddleLayerOpsPaths(
|
|
285
329
|
workDir: string,
|
|
286
330
|
): Pick<
|
|
287
331
|
MiddleLayerOpsPaths,
|
|
288
|
-
keyof ReturnType<typeof DefaultDriverKitOpsPaths> | "frontendDownloadPath"
|
|
332
|
+
keyof ReturnType<typeof DefaultDriverKitOpsPaths> | "frontendDownloadPath" | "treeSnapshotPath"
|
|
289
333
|
> {
|
|
290
334
|
return {
|
|
291
335
|
...DefaultDriverKitOpsPaths(workDir),
|
|
292
336
|
frontendDownloadPath: path.join(workDir, "frontend"),
|
|
337
|
+
treeSnapshotPath: path.join(workDir, "treeSnapshots"),
|
|
293
338
|
};
|
|
294
339
|
}
|
|
295
340
|
|