@elaraai/e3-ui-cli 1.0.77 → 1.0.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/assets/{engine-DAHqaY4g-BjS4_bVx.js → engine-DAHqaY4g-B9nTMQVT.js} +1 -1
- package/dist/app/assets/{index-B9hTRqaB.js → index-CIIzwdrV.js} +101 -101
- package/dist/app/index.html +1 -1
- package/dist/tui/data/dataset.d.ts +54 -17
- package/dist/tui/data/dataset.d.ts.map +1 -1
- package/dist/tui/data/dataset.js +291 -98
- package/dist/tui/data/dataset.js.map +1 -1
- package/dist/tui/model/tree.d.ts +0 -2
- package/dist/tui/model/tree.d.ts.map +1 -1
- package/dist/tui/model/tree.js +1 -3
- package/dist/tui/model/tree.js.map +1 -1
- package/dist/tui/state/actions.d.ts +4 -1
- package/dist/tui/state/actions.d.ts.map +1 -1
- package/dist/tui/state/actions.js.map +1 -1
- package/dist/tui/ui/widgets/tree.d.ts.map +1 -1
- package/dist/tui/ui/widgets/tree.js +4 -3
- package/dist/tui/ui/widgets/tree.js.map +1 -1
- package/package.json +14 -14
package/dist/app/index.html
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
<meta charset="UTF-8" />
|
|
5
5
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
6
6
|
<title>e3-ui shot</title>
|
|
7
|
-
<script type="module" crossorigin src="./assets/index-
|
|
7
|
+
<script type="module" crossorigin src="./assets/index-CIIzwdrV.js"></script>
|
|
8
8
|
<link rel="stylesheet" crossorigin href="./assets/index-CYR-0Grb.css">
|
|
9
9
|
</head>
|
|
10
10
|
<body>
|
|
@@ -8,35 +8,62 @@
|
|
|
8
8
|
* `tick` polls `datasetGetStatus` (every 5 s while the dataset is on
|
|
9
9
|
* screen) and, on a new content hash, decides how the value is shown, as
|
|
10
10
|
* the web preview does: no value → *unset* / *null*; a collection root
|
|
11
|
-
* (Array / Set / Dict) → *paged* through `datasetGetPage
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
11
|
+
* (Array / Set / Dict) → *paged* through `datasetGetPage`, read-only, at
|
|
12
|
+
* every size; any other root ≤ 200 KB → fetched whole, decoded and
|
|
13
|
+
* materialized into the tree (*inline*); larger → *too large* (`/save`
|
|
14
|
+
* still works). The server's refusals map to states: `dataset_not_indexed`
|
|
15
|
+
* (a legacy blob) → *not indexed* with a `⏎ load whole value` fallback up
|
|
16
|
+
* to 64 MB, `dataset_too_large` → *too large*, `dataset_hash_mismatch` →
|
|
17
|
+
* the status is refetched.
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
19
|
+
* Paging is driven by one *window* per shown dataset — the pages the view
|
|
20
|
+
* wants right now, set by every scroll ({@link DatasetLoader.needRows}) —
|
|
21
|
+
* not by a queue of requests: at most {@link MAX_INFLIGHT} fetches run per
|
|
22
|
+
* dataset, each issued for a page of the current window (nearest its
|
|
23
|
+
* centre first), so a thumb drag across a million rows costs the pages of
|
|
24
|
+
* where it stops, not of everywhere it passed. A page's size in rows is
|
|
25
|
+
* chosen per dataset from the blob's stored bytes per row
|
|
26
|
+
* ({@link pageSizeFor}): a page of wide rows holds as many rows as
|
|
27
|
+
* {@link PAGE_BYTES_TARGET} covers, a page of narrow rows
|
|
28
|
+
* {@link PAGE_SIZE_MAX}; a server that cuts a window short by its own byte
|
|
29
|
+
* budget is asked for the rest until the page is whole. Loaded pages are
|
|
30
|
+
* pruned to {@link MAX_RETAINED_PAGES} around the window whenever the set
|
|
31
|
+
* changes — as a page lands as much as when the window moves — and raw
|
|
32
|
+
* bytes stay in a small cache so a page that left re-materializes without
|
|
33
|
+
* a request. Decoding and materializing yield to the renderer.
|
|
23
34
|
*
|
|
24
35
|
* @packageDocumentation
|
|
25
36
|
*/
|
|
26
37
|
import { type EastTypeValue } from '@elaraai/east';
|
|
27
38
|
import { type DatasetKeyMatchRange, type DatasetKeyQuery, type ValueTreePagedRow } from '@elaraai/east-ui/internal';
|
|
28
39
|
import { type Api } from '../api.js';
|
|
29
|
-
import {
|
|
40
|
+
import { collectionKeys, isCollectionType, keyTypeOf, viewDataset } from '../model/tree.js';
|
|
30
41
|
import type { Store } from '../state/store.js';
|
|
31
|
-
export {
|
|
42
|
+
export { collectionKeys, isCollectionType, keyTypeOf, viewDataset };
|
|
32
43
|
/** Non-collection values up to this many bytes are fetched whole. */
|
|
33
44
|
export declare const INLINE_LIMIT: number;
|
|
34
45
|
/** `⏎ load whole value` on a not-indexed collection is offered up to this size. */
|
|
35
46
|
export declare const WHOLE_LIMIT: number;
|
|
36
|
-
/**
|
|
37
|
-
export declare const
|
|
47
|
+
/** Root rows per page at most — the page size of narrow rows. */
|
|
48
|
+
export declare const PAGE_SIZE_MAX = 500;
|
|
49
|
+
/** Stored bytes a page aims to hold: wide rows make shorter pages, so what one page materializes stays bounded. */
|
|
50
|
+
export declare const PAGE_BYTES_TARGET: number;
|
|
51
|
+
/** Loaded (materialized) pages retained around the window. */
|
|
52
|
+
export declare const MAX_RETAINED_PAGES = 6;
|
|
38
53
|
/** Raw page bytes kept per session (a return re-materializes without a request). */
|
|
39
54
|
export declare const MAX_CACHED_PAGES = 32;
|
|
55
|
+
/** Page fetches in flight per dataset. */
|
|
56
|
+
export declare const MAX_INFLIGHT = 2;
|
|
57
|
+
/**
|
|
58
|
+
* The rows per page of a collection: as many as {@link PAGE_BYTES_TARGET}
|
|
59
|
+
* covers at the blob's average stored bytes per row, at most
|
|
60
|
+
* {@link PAGE_SIZE_MAX}, at least one.
|
|
61
|
+
*
|
|
62
|
+
* @param totalBytes - The stored blob's size
|
|
63
|
+
* @param totalRows - Its root rows
|
|
64
|
+
* @returns The page size
|
|
65
|
+
*/
|
|
66
|
+
export declare function pageSizeFor(totalBytes: number, totalRows: number): number;
|
|
40
67
|
/**
|
|
41
68
|
* Materializes one decoded page into the tree's paged-row contract (the
|
|
42
69
|
* web preview's `pageRows`).
|
|
@@ -52,6 +79,10 @@ export interface DatasetLoaderDeps {
|
|
|
52
79
|
store: Store;
|
|
53
80
|
api: () => Api | null;
|
|
54
81
|
log?: ((line: string) => void) | undefined;
|
|
82
|
+
/** The clock (for the retry hold). */
|
|
83
|
+
now?: (() => number) | undefined;
|
|
84
|
+
/** How long a failed page waits before it is asked for again. */
|
|
85
|
+
retryAfterMs?: number | undefined;
|
|
55
86
|
}
|
|
56
87
|
/** The dataset loader. */
|
|
57
88
|
export interface DatasetLoader {
|
|
@@ -59,7 +90,13 @@ export interface DatasetLoader {
|
|
|
59
90
|
tick(ws: string, path: string, force?: boolean): Promise<void>;
|
|
60
91
|
/** Reloads the content now, past any conflict (`/reload`). */
|
|
61
92
|
reload(ws: string, path: string): Promise<void>;
|
|
62
|
-
/**
|
|
93
|
+
/**
|
|
94
|
+
* Sets the window: the pages covering root rows `[startRow, endRow)` are
|
|
95
|
+
* what the view wants now. They are fetched two at a time, nearest the
|
|
96
|
+
* window's centre first; pages far from them are dropped; a window set
|
|
97
|
+
* while pages are in flight replaces the last one, so what lands for a
|
|
98
|
+
* position the view has left is cached, not shown.
|
|
99
|
+
*/
|
|
63
100
|
needRows(ws: string, path: string, startRow: number, endRow: number): void;
|
|
64
101
|
/** `⏎ load whole value` on a not-indexed collection. */
|
|
65
102
|
loadWhole(ws: string, path: string): Promise<void>;
|
|
@@ -67,7 +104,7 @@ export interface DatasetLoader {
|
|
|
67
104
|
findKey(ws: string, path: string, query: DatasetKeyQuery): Promise<DatasetKeyMatchRange>;
|
|
68
105
|
/** The stored bytes (`/save`). */
|
|
69
106
|
bytes(ws: string, path: string): Promise<Uint8Array>;
|
|
70
|
-
/** Drops the caches and in-flight bookkeeping. */
|
|
107
|
+
/** Drops the caches, the windows and the in-flight bookkeeping. */
|
|
71
108
|
reset(): void;
|
|
72
109
|
}
|
|
73
110
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../../src/tui/data/dataset.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH
|
|
1
|
+
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../../src/tui/data/dataset.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAA4B,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAE7E,OAAO,EAAqC,KAAK,oBAAoB,EAAE,KAAK,eAAe,EAAE,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AACvJ,OAAO,EAAsC,KAAK,GAAG,EAAE,MAAM,WAAW,CAAC;AACzE,OAAO,EAA8B,cAAc,EAAE,gBAAgB,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAExH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;AAEpE,qEAAqE;AACrE,eAAO,MAAM,YAAY,QAAa,CAAC;AACvC,mFAAmF;AACnF,eAAO,MAAM,WAAW,QAAmB,CAAC;AAC5C,iEAAiE;AACjE,eAAO,MAAM,aAAa,MAAM,CAAC;AACjC,mHAAmH;AACnH,eAAO,MAAM,iBAAiB,QAAa,CAAC;AAC5C,8DAA8D;AAC9D,eAAO,MAAM,kBAAkB,IAAI,CAAC;AACpC,oFAAoF;AACpF,eAAO,MAAM,gBAAgB,KAAK,CAAC;AACnC,0CAA0C;AAC1C,eAAO,MAAM,YAAY,IAAI,CAAC;AAW9B;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAGzE;AAwBD;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,iBAAiB,EAAE,CAEnG;AAED,6BAA6B;AAC7B,MAAM,WAAW,iBAAiB;IAC9B,KAAK,EAAE,KAAK,CAAC;IACb,GAAG,EAAE,MAAM,GAAG,GAAG,IAAI,CAAC;IACtB,GAAG,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC;IAC3C,sCAAsC;IACtC,GAAG,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;IACjC,iEAAiE;IACjE,YAAY,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,0BAA0B;AAC1B,MAAM,WAAW,aAAa;IAC1B,8GAA8G;IAC9G,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,8DAA8D;IAC9D,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3E,wDAAwD;IACxD,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnD,uFAAuF;IACvF,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACzF,kCAAkC;IAClC,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IACrD,mEAAmE;IACnE,KAAK,IAAI,IAAI,CAAC;CACjB;AAaD;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,aAAa,CAkV1E"}
|
package/dist/tui/data/dataset.js
CHANGED
|
@@ -8,18 +8,29 @@
|
|
|
8
8
|
* `tick` polls `datasetGetStatus` (every 5 s while the dataset is on
|
|
9
9
|
* screen) and, on a new content hash, decides how the value is shown, as
|
|
10
10
|
* the web preview does: no value → *unset* / *null*; a collection root
|
|
11
|
-
* (Array / Set / Dict) → *paged* through `datasetGetPage
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
11
|
+
* (Array / Set / Dict) → *paged* through `datasetGetPage`, read-only, at
|
|
12
|
+
* every size; any other root ≤ 200 KB → fetched whole, decoded and
|
|
13
|
+
* materialized into the tree (*inline*); larger → *too large* (`/save`
|
|
14
|
+
* still works). The server's refusals map to states: `dataset_not_indexed`
|
|
15
|
+
* (a legacy blob) → *not indexed* with a `⏎ load whole value` fallback up
|
|
16
|
+
* to 64 MB, `dataset_too_large` → *too large*, `dataset_hash_mismatch` →
|
|
17
|
+
* the status is refetched.
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
19
|
+
* Paging is driven by one *window* per shown dataset — the pages the view
|
|
20
|
+
* wants right now, set by every scroll ({@link DatasetLoader.needRows}) —
|
|
21
|
+
* not by a queue of requests: at most {@link MAX_INFLIGHT} fetches run per
|
|
22
|
+
* dataset, each issued for a page of the current window (nearest its
|
|
23
|
+
* centre first), so a thumb drag across a million rows costs the pages of
|
|
24
|
+
* where it stops, not of everywhere it passed. A page's size in rows is
|
|
25
|
+
* chosen per dataset from the blob's stored bytes per row
|
|
26
|
+
* ({@link pageSizeFor}): a page of wide rows holds as many rows as
|
|
27
|
+
* {@link PAGE_BYTES_TARGET} covers, a page of narrow rows
|
|
28
|
+
* {@link PAGE_SIZE_MAX}; a server that cuts a window short by its own byte
|
|
29
|
+
* budget is asked for the rest until the page is whole. Loaded pages are
|
|
30
|
+
* pruned to {@link MAX_RETAINED_PAGES} around the window whenever the set
|
|
31
|
+
* changes — as a page lands as much as when the window moves — and raw
|
|
32
|
+
* bytes stay in a small cache so a page that left re-materializes without
|
|
33
|
+
* a request. Decoding and materializing yield to the renderer.
|
|
23
34
|
*
|
|
24
35
|
* @packageDocumentation
|
|
25
36
|
*/
|
|
@@ -27,18 +38,64 @@ import { decodeBeast2For, variant } from '@elaraai/east';
|
|
|
27
38
|
import { ValueTree } from '@elaraai/east-ui';
|
|
28
39
|
import { findKeyInline, pruneRetainedPages } from '@elaraai/east-ui/internal';
|
|
29
40
|
import { apiCode, describeError, treePathOf } from '../api.js';
|
|
30
|
-
import {
|
|
31
|
-
export {
|
|
41
|
+
import { applyAnchor, captureAnchor, collectionKeys, isCollectionType, keyTypeOf, viewDataset } from '../model/tree.js';
|
|
42
|
+
export { collectionKeys, isCollectionType, keyTypeOf, viewDataset };
|
|
32
43
|
/** Non-collection values up to this many bytes are fetched whole. */
|
|
33
44
|
export const INLINE_LIMIT = 200 * 1024;
|
|
34
45
|
/** `⏎ load whole value` on a not-indexed collection is offered up to this size. */
|
|
35
46
|
export const WHOLE_LIMIT = 64 * 1024 * 1024;
|
|
36
|
-
/**
|
|
37
|
-
export const
|
|
47
|
+
/** Root rows per page at most — the page size of narrow rows. */
|
|
48
|
+
export const PAGE_SIZE_MAX = 500;
|
|
49
|
+
/** Stored bytes a page aims to hold: wide rows make shorter pages, so what one page materializes stays bounded. */
|
|
50
|
+
export const PAGE_BYTES_TARGET = 128 * 1024;
|
|
51
|
+
/** Loaded (materialized) pages retained around the window. */
|
|
52
|
+
export const MAX_RETAINED_PAGES = 6;
|
|
38
53
|
/** Raw page bytes kept per session (a return re-materializes without a request). */
|
|
39
54
|
export const MAX_CACHED_PAGES = 32;
|
|
55
|
+
/** Page fetches in flight per dataset. */
|
|
56
|
+
export const MAX_INFLIGHT = 2;
|
|
40
57
|
/** Root rows requested around the first window before the view has scrolled. */
|
|
41
58
|
const INITIAL_WINDOW_ROWS = 60;
|
|
59
|
+
/** Rows materialized between yields to the renderer. */
|
|
60
|
+
const MATERIALIZE_SLICE = 100;
|
|
61
|
+
/** How long a failed page waits before it is asked for again. */
|
|
62
|
+
const RETRY_AFTER_MS = 2_000;
|
|
63
|
+
/**
|
|
64
|
+
* The rows per page of a collection: as many as {@link PAGE_BYTES_TARGET}
|
|
65
|
+
* covers at the blob's average stored bytes per row, at most
|
|
66
|
+
* {@link PAGE_SIZE_MAX}, at least one.
|
|
67
|
+
*
|
|
68
|
+
* @param totalBytes - The stored blob's size
|
|
69
|
+
* @param totalRows - Its root rows
|
|
70
|
+
* @returns The page size
|
|
71
|
+
*/
|
|
72
|
+
export function pageSizeFor(totalBytes, totalRows) {
|
|
73
|
+
if (totalRows <= 0 || totalBytes <= 0)
|
|
74
|
+
return PAGE_SIZE_MAX;
|
|
75
|
+
return Math.max(1, Math.min(PAGE_SIZE_MAX, Math.floor(PAGE_BYTES_TARGET / (totalBytes / totalRows))));
|
|
76
|
+
}
|
|
77
|
+
/** The elements of a decoded page in row order (a Dict's entries as pairs). */
|
|
78
|
+
function elementsOf(type, decoded) {
|
|
79
|
+
if (type.type === 'Array')
|
|
80
|
+
return decoded;
|
|
81
|
+
if (type.type === 'Dict')
|
|
82
|
+
return Array.from(decoded.entries());
|
|
83
|
+
return Array.from(decoded.values());
|
|
84
|
+
}
|
|
85
|
+
/** One root row of the paged tree: the element materialized, with its global step and label. */
|
|
86
|
+
function rowOf(type, element, row) {
|
|
87
|
+
if (type.type === 'Dict') {
|
|
88
|
+
const { key: keyType, value: valueType } = type.value;
|
|
89
|
+
const [k, v] = element;
|
|
90
|
+
const stringKeys = keyType.type === 'String';
|
|
91
|
+
return {
|
|
92
|
+
node: ValueTree.materialize(valueType, v),
|
|
93
|
+
step: stringKeys ? variant('key', k) : variant('index', BigInt(row)),
|
|
94
|
+
label: stringKeys ? k : ValueTree.keyLabel(keyType, k),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return { node: ValueTree.materialize(type.value, element), step: variant('index', BigInt(row)) };
|
|
98
|
+
}
|
|
42
99
|
/**
|
|
43
100
|
* Materializes one decoded page into the tree's paged-row contract (the
|
|
44
101
|
* web preview's `pageRows`).
|
|
@@ -49,23 +106,11 @@ const INITIAL_WINDOW_ROWS = 60;
|
|
|
49
106
|
* @returns The rows
|
|
50
107
|
*/
|
|
51
108
|
export function pageRows(type, decoded, offset) {
|
|
52
|
-
|
|
53
|
-
const elemType = type.value;
|
|
54
|
-
return decoded.map((el, i) => ({ node: ValueTree.materialize(elemType, el), step: variant('index', BigInt(offset + i)) }));
|
|
55
|
-
}
|
|
56
|
-
if (type.type === 'Dict') {
|
|
57
|
-
const { key: keyType, value: valueType } = type.value;
|
|
58
|
-
const stringKeys = keyType.type === 'String';
|
|
59
|
-
return Array.from(decoded.entries()).map(([k, v], i) => ({
|
|
60
|
-
node: ValueTree.materialize(valueType, v),
|
|
61
|
-
step: stringKeys ? variant('key', k) : variant('index', BigInt(offset + i)),
|
|
62
|
-
label: stringKeys ? k : ValueTree.keyLabel(keyType, k),
|
|
63
|
-
}));
|
|
64
|
-
}
|
|
65
|
-
const elemType = type.value;
|
|
66
|
-
return Array.from(decoded.values()).map((el, i) => ({ node: ValueTree.materialize(elemType, el), step: variant('index', BigInt(offset + i)) }));
|
|
109
|
+
return elementsOf(type, decoded).map((element, i) => rowOf(type, element, offset + i));
|
|
67
110
|
}
|
|
68
111
|
const yieldToRender = () => new Promise(resolve => setImmediate(resolve));
|
|
112
|
+
const chunkRows = (entry) => entry.chunks.reduce((n, c) => n + c.count, 0);
|
|
113
|
+
const sameList = (a, b) => a.length === b.length && a.every((x, i) => x === b[i]);
|
|
69
114
|
/**
|
|
70
115
|
* Creates the loader.
|
|
71
116
|
*
|
|
@@ -74,11 +119,31 @@ const yieldToRender = () => new Promise(resolve => setImmediate(resolve));
|
|
|
74
119
|
*/
|
|
75
120
|
export function createDatasetLoader(deps) {
|
|
76
121
|
const { store } = deps;
|
|
77
|
-
const
|
|
122
|
+
const clock = deps.now ?? Date.now;
|
|
123
|
+
const retryAfterMs = deps.retryAfterMs ?? RETRY_AFTER_MS;
|
|
124
|
+
const wholeInflight = new Set();
|
|
78
125
|
const cache = new Map();
|
|
126
|
+
/** Per dataset (`ws\npath`): the window. */
|
|
127
|
+
const windows = new Map();
|
|
128
|
+
/** Per dataset and hash: the pages being fetched. */
|
|
129
|
+
const inflight = new Map();
|
|
130
|
+
/** Per page key: when a failed page may be asked for again. */
|
|
131
|
+
const retryAt = new Map();
|
|
132
|
+
/** Per dataset: the pump already scheduled for a hold, and when for. */
|
|
133
|
+
const retryPump = new Map();
|
|
134
|
+
const datasetKey = (ws, path) => `${ws}\n${path}`;
|
|
79
135
|
const keyOf = (ws, path, hash, page) => `${ws}\n${path}\n${hash}\n${page}`;
|
|
80
136
|
const current = (ws, path) => store.getState().data.dataset[ws]?.[path];
|
|
81
137
|
const notFound = { found: false, row: 0, count: 0 };
|
|
138
|
+
const inflightOf = (ws, path, hash) => {
|
|
139
|
+
const key = keyOf(ws, path, hash, 'whole');
|
|
140
|
+
let set = inflight.get(key);
|
|
141
|
+
if (set === undefined) {
|
|
142
|
+
set = new Set();
|
|
143
|
+
inflight.set(key, set);
|
|
144
|
+
}
|
|
145
|
+
return set;
|
|
146
|
+
};
|
|
82
147
|
/** Replaces the mode, unless the value changed underneath; the shown tree's selection stays on its logical row. */
|
|
83
148
|
const setMode = (ws, path, hash, mode) => {
|
|
84
149
|
const s = store.getState();
|
|
@@ -103,11 +168,20 @@ export function createDatasetLoader(deps) {
|
|
|
103
168
|
cache.delete(oldest);
|
|
104
169
|
}
|
|
105
170
|
};
|
|
171
|
+
/** The window's pages not loaded yet, nearest its centre first. */
|
|
172
|
+
const wantedMissing = (w, mode) => {
|
|
173
|
+
const centre = (w.first + w.last) / 2;
|
|
174
|
+
const missing = [];
|
|
175
|
+
for (let p = w.first; p <= w.last; p++)
|
|
176
|
+
if (!mode.pages.has(p))
|
|
177
|
+
missing.push(p);
|
|
178
|
+
return missing.sort((a, b) => Math.abs(a - centre) - Math.abs(b - centre) || a - b);
|
|
179
|
+
};
|
|
106
180
|
const loadInline = async (api, ws, path, hash, type) => {
|
|
107
181
|
const key = keyOf(ws, path, hash, 'whole');
|
|
108
|
-
if (
|
|
182
|
+
if (wholeInflight.has(key))
|
|
109
183
|
return;
|
|
110
|
-
|
|
184
|
+
wholeInflight.add(key);
|
|
111
185
|
try {
|
|
112
186
|
const got = await api.datasetGet(ws, treePathOf(path));
|
|
113
187
|
await yieldToRender();
|
|
@@ -121,63 +195,169 @@ export function createDatasetLoader(deps) {
|
|
|
121
195
|
setMode(ws, path, hash, { kind: 'error', message: describeError(err) });
|
|
122
196
|
}
|
|
123
197
|
finally {
|
|
124
|
-
|
|
198
|
+
wholeInflight.delete(key);
|
|
125
199
|
}
|
|
126
200
|
};
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
201
|
+
/** Maps a page request's failure to a state; a paged value holds the page back before it is asked for again. */
|
|
202
|
+
const pageFailure = (ws, path, hash, page, err) => {
|
|
203
|
+
const code = apiCode(err);
|
|
204
|
+
const d = current(ws, path);
|
|
205
|
+
if (code === 'dataset_hash_mismatch') {
|
|
206
|
+
deps.log?.(`dataset ${ws}${path} page ${page}: hash mismatch — refetching the status`);
|
|
207
|
+
void loader.tick(ws, path);
|
|
208
|
+
}
|
|
209
|
+
else if (code === 'dataset_not_indexed') {
|
|
210
|
+
setMode(ws, path, hash, { kind: 'not-indexed', loadable: (d?.size ?? 0) <= WHOLE_LIMIT });
|
|
211
|
+
}
|
|
212
|
+
else if (code === 'dataset_too_large') {
|
|
213
|
+
setMode(ws, path, hash, { kind: 'too-large' });
|
|
214
|
+
}
|
|
215
|
+
else if (d?.mode.kind === 'paged' && d.hash === hash) {
|
|
216
|
+
deps.log?.(`dataset ${ws}${path} page ${page} failed: ${describeError(err)}`);
|
|
217
|
+
// A failing server is not hammered: the page waits out the hold,
|
|
218
|
+
// then the window's next move or the hold's end asks again. The
|
|
219
|
+
// pump below re-arms for the hold, so the wait cannot be lost.
|
|
220
|
+
retryAt.set(keyOf(ws, path, hash, page), clock() + retryAfterMs);
|
|
221
|
+
}
|
|
222
|
+
else {
|
|
223
|
+
deps.log?.(`dataset ${ws}${path} first page failed: ${describeError(err)}`);
|
|
224
|
+
setMode(ws, path, hash, { kind: 'error', message: describeError(err) });
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
/** Fetches the rest of a page — window by window when the server cuts one short — into the byte cache. */
|
|
228
|
+
const fetchPage = async (api, ws, path, hash, key, start, want, partial) => {
|
|
229
|
+
const chunks = partial === undefined ? [] : [...partial.chunks];
|
|
230
|
+
let got = chunkRows({ chunks });
|
|
231
|
+
while (got < want) {
|
|
232
|
+
const p = await api.datasetGetPage(ws, treePathOf(path), { offset: start + got, limit: want - got, hash });
|
|
233
|
+
const count = Math.max(0, Math.min(p.count, want - got));
|
|
234
|
+
chunks.push({ offset: start + got, count, bytes: p.data });
|
|
235
|
+
if (count === 0)
|
|
236
|
+
break; // nothing more came: the page ends here
|
|
237
|
+
got += count;
|
|
238
|
+
}
|
|
239
|
+
const entry = { chunks };
|
|
240
|
+
cachePut(key, entry);
|
|
241
|
+
return entry;
|
|
242
|
+
};
|
|
243
|
+
/** Decodes a page's chunks and materializes its rows `[start, start + want)`, yielding as it goes. */
|
|
244
|
+
const materializePage = async (type, entry, start, want) => {
|
|
245
|
+
const rows = [];
|
|
246
|
+
const decode = decodeBeast2For(type);
|
|
247
|
+
for (const chunk of entry.chunks) {
|
|
248
|
+
const elements = elementsOf(type, decode(chunk.bytes));
|
|
249
|
+
for (let i = 0; i < elements.length; i++) {
|
|
250
|
+
const row = chunk.offset + i;
|
|
251
|
+
if (row < start || row >= start + want)
|
|
252
|
+
continue;
|
|
253
|
+
rows.push(rowOf(type, elements[i], row));
|
|
254
|
+
if (rows.length % MATERIALIZE_SLICE === 0)
|
|
255
|
+
await yieldToRender();
|
|
256
|
+
}
|
|
135
257
|
}
|
|
258
|
+
return rows;
|
|
259
|
+
};
|
|
260
|
+
/** Loads one page of the window: its bytes (cached or fetched), then its rows — unless the window left it behind meanwhile. */
|
|
261
|
+
const loadPage = async (api, ws, path, hash, type, mode, page) => {
|
|
262
|
+
const running = inflightOf(ws, path, hash);
|
|
263
|
+
if (running.has(page))
|
|
264
|
+
return;
|
|
265
|
+
running.add(page);
|
|
266
|
+
const key = keyOf(ws, path, hash, page);
|
|
136
267
|
try {
|
|
268
|
+
const start = page * mode.pageSize;
|
|
269
|
+
const want = Math.max(0, Math.min(mode.pageSize, mode.totalRows - start));
|
|
137
270
|
let entry = cache.get(key);
|
|
138
|
-
if (entry === undefined)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
271
|
+
if (entry === undefined || chunkRows(entry) < want)
|
|
272
|
+
entry = await fetchPage(api, ws, path, hash, key, start, want, entry);
|
|
273
|
+
// The window moved on while the page was in flight: its bytes are
|
|
274
|
+
// cached for a return; nothing is materialized for a page nobody shows.
|
|
275
|
+
const w = windows.get(datasetKey(ws, path));
|
|
276
|
+
if (w === undefined || w.hash !== hash || page < w.first || page > w.last)
|
|
277
|
+
return;
|
|
143
278
|
await yieldToRender();
|
|
144
|
-
const rows =
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
279
|
+
const rows = await materializePage(type, entry, start, want);
|
|
280
|
+
const d = current(ws, path);
|
|
281
|
+
if (d === undefined || d.hash !== hash || d.mode.kind !== 'paged')
|
|
147
282
|
return;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
283
|
+
const after = windows.get(datasetKey(ws, path));
|
|
284
|
+
const pages = new Map(d.mode.pages);
|
|
285
|
+
pages.set(page, rows);
|
|
286
|
+
const kept = after !== undefined && after.hash === hash ? pruneRetainedPages(pages, after.first, after.last, MAX_RETAINED_PAGES) : pages;
|
|
287
|
+
const next = { ...d.mode, pages: kept, loading: [] };
|
|
288
|
+
if (after !== undefined && after.hash === hash)
|
|
289
|
+
next.loading = wantedMissing(after, next);
|
|
290
|
+
setMode(ws, path, hash, next);
|
|
156
291
|
}
|
|
157
292
|
catch (err) {
|
|
158
|
-
|
|
159
|
-
const now = current(ws, path);
|
|
160
|
-
if (code === 'dataset_hash_mismatch') {
|
|
161
|
-
deps.log?.(`dataset ${ws}${path} page ${page}: hash mismatch — refetching the status`);
|
|
162
|
-
void loader.tick(ws, path);
|
|
163
|
-
}
|
|
164
|
-
else if (code === 'dataset_not_indexed') {
|
|
165
|
-
setMode(ws, path, hash, { kind: 'not-indexed', loadable: (now?.size ?? 0) <= WHOLE_LIMIT });
|
|
166
|
-
}
|
|
167
|
-
else if (code === 'dataset_too_large') {
|
|
168
|
-
setMode(ws, path, hash, { kind: 'too-large' });
|
|
169
|
-
}
|
|
170
|
-
else if (now?.mode.kind === 'paged') {
|
|
171
|
-
deps.log?.(`dataset ${ws}${path} page ${page} failed: ${describeError(err)}`);
|
|
172
|
-
setMode(ws, path, hash, { ...now.mode, loading: now.mode.loading.filter(p => p !== page) });
|
|
173
|
-
}
|
|
174
|
-
else {
|
|
175
|
-
deps.log?.(`dataset ${ws}${path} first page failed: ${describeError(err)}`);
|
|
176
|
-
setMode(ws, path, hash, { kind: 'error', message: describeError(err) });
|
|
177
|
-
}
|
|
293
|
+
pageFailure(ws, path, hash, page, err);
|
|
178
294
|
}
|
|
179
295
|
finally {
|
|
180
|
-
|
|
296
|
+
running.delete(page);
|
|
297
|
+
pump(ws, path);
|
|
298
|
+
}
|
|
299
|
+
};
|
|
300
|
+
/** Starts loads for the window's missing pages, up to the in-flight cap. */
|
|
301
|
+
const pump = (ws, path) => {
|
|
302
|
+
const api = deps.api();
|
|
303
|
+
const w = windows.get(datasetKey(ws, path));
|
|
304
|
+
const d = current(ws, path);
|
|
305
|
+
if (api === null || w === undefined || d === undefined || d.hash !== w.hash || d.type === null || d.mode.kind !== 'paged')
|
|
306
|
+
return;
|
|
307
|
+
const running = inflightOf(ws, path, w.hash);
|
|
308
|
+
const now = clock();
|
|
309
|
+
let soonestHold = Infinity;
|
|
310
|
+
for (const page of wantedMissing(w, d.mode)) {
|
|
311
|
+
if (running.size >= MAX_INFLIGHT)
|
|
312
|
+
break;
|
|
313
|
+
if (running.has(page))
|
|
314
|
+
continue;
|
|
315
|
+
const hold = retryAt.get(keyOf(ws, path, w.hash, page));
|
|
316
|
+
if (hold !== undefined && hold > now) {
|
|
317
|
+
soonestHold = Math.min(soonestHold, hold);
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
void loadPage(api, ws, path, w.hash, d.type, d.mode, page);
|
|
321
|
+
}
|
|
322
|
+
// A page held back is asked for again when its hold ends — by THIS
|
|
323
|
+
// pump, which is the only thing that will. Scheduling the wait where
|
|
324
|
+
// the failure happened was not enough: `setTimeout` counts on libuv's
|
|
325
|
+
// monotonic clock and the hold is stamped from `Date.now`, so a timer
|
|
326
|
+
// that fires a millisecond before the wall clock agrees found the page
|
|
327
|
+
// still held, skipped it, and left nothing to ask again — the page
|
|
328
|
+
// stranded until the window next moved. Re-arming here closes that,
|
|
329
|
+
// whatever the two clocks think of each other.
|
|
330
|
+
if (soonestHold !== Infinity)
|
|
331
|
+
armRetryPump(ws, path, soonestHold, now);
|
|
332
|
+
};
|
|
333
|
+
/** Schedules the pump that ends the soonest hold, keeping one timer per
|
|
334
|
+
* dataset: an earlier wait replaces a later one, a later one is already
|
|
335
|
+
* covered. */
|
|
336
|
+
const armRetryPump = (ws, path, at, now) => {
|
|
337
|
+
const key = datasetKey(ws, path);
|
|
338
|
+
const pending = retryPump.get(key);
|
|
339
|
+
if (pending !== undefined) {
|
|
340
|
+
if (pending.at <= at)
|
|
341
|
+
return;
|
|
342
|
+
clearTimeout(pending.timer);
|
|
343
|
+
}
|
|
344
|
+
const timer = setTimeout(() => {
|
|
345
|
+
retryPump.delete(key);
|
|
346
|
+
pump(ws, path);
|
|
347
|
+
}, Math.max(1, at - now));
|
|
348
|
+
timer.unref?.();
|
|
349
|
+
retryPump.set(key, { at, timer });
|
|
350
|
+
};
|
|
351
|
+
/** A server that reports no row geometry: one window of the head tells the totals, and its rows seed page 0. */
|
|
352
|
+
const probe = async (api, ws, path, hash) => {
|
|
353
|
+
try {
|
|
354
|
+
const p = await api.datasetGetPage(ws, treePathOf(path), { offset: 0, limit: PAGE_SIZE_MAX, hash });
|
|
355
|
+
cachePut(keyOf(ws, path, hash, 0), { chunks: [{ offset: 0, count: p.count, bytes: p.data }] });
|
|
356
|
+
return { kind: 'paged', pageSize: pageSizeFor(p.totalBytes, p.totalElements), totalRows: p.totalElements, totalBytes: p.totalBytes, pages: new Map(), loading: [] };
|
|
357
|
+
}
|
|
358
|
+
catch (err) {
|
|
359
|
+
pageFailure(ws, path, hash, 0, err);
|
|
360
|
+
return null;
|
|
181
361
|
}
|
|
182
362
|
};
|
|
183
363
|
const loader = {
|
|
@@ -210,12 +390,20 @@ export function createDatasetLoader(deps) {
|
|
|
210
390
|
}
|
|
211
391
|
store.dispatch({ type: 'data/dataset', ws, path, data: base });
|
|
212
392
|
if (isCollectionType(type)) {
|
|
213
|
-
|
|
393
|
+
// The status carries the stored geometry, so the page size is known
|
|
394
|
+
// before a row is fetched; a server without it is probed once.
|
|
395
|
+
const rows = status.rows.type === 'some' ? Number(status.rows.value) : null;
|
|
396
|
+
const paged = rows !== null
|
|
397
|
+
? { kind: 'paged', pageSize: pageSizeFor(size, rows), totalRows: rows, totalBytes: size, pages: new Map(), loading: [] }
|
|
398
|
+
: await probe(api, ws, path, hash);
|
|
399
|
+
if (paged === null)
|
|
400
|
+
return;
|
|
401
|
+
setMode(ws, path, hash, paged);
|
|
214
402
|
// The first window around wherever the view sits (a restored top row may be deep).
|
|
215
403
|
const s = store.getState();
|
|
216
404
|
const shown = viewDataset(s);
|
|
217
405
|
const top = shown !== null && shown.ws === ws && shown.path === path && (s.view.kind === 'task' || s.view.kind === 'input') ? s.view.tree.top : 0;
|
|
218
|
-
loader.needRows(ws, path, Math.max(0, top -
|
|
406
|
+
loader.needRows(ws, path, Math.max(0, top - paged.pageSize), top + INITIAL_WINDOW_ROWS + paged.pageSize);
|
|
219
407
|
}
|
|
220
408
|
else if (size <= INLINE_LIMIT) {
|
|
221
409
|
await loadInline(api, ws, path, hash, type);
|
|
@@ -225,29 +413,28 @@ export function createDatasetLoader(deps) {
|
|
|
225
413
|
}
|
|
226
414
|
},
|
|
227
415
|
needRows(ws, path, startRow, endRow) {
|
|
228
|
-
const api = deps.api();
|
|
229
416
|
const d = current(ws, path);
|
|
230
|
-
if (
|
|
417
|
+
if (d === undefined || d.mode.kind !== 'paged' || d.hash === null || d.type === null)
|
|
418
|
+
return;
|
|
419
|
+
const { pageSize, totalRows } = d.mode;
|
|
420
|
+
const pageCount = Math.ceil(totalRows / pageSize);
|
|
421
|
+
if (pageCount === 0)
|
|
231
422
|
return;
|
|
232
|
-
const
|
|
233
|
-
const
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
if (pruned !== d.mode.pages)
|
|
242
|
-
setMode(ws, path, d.hash, { ...d.mode, pages: pruned });
|
|
243
|
-
for (const p of missing)
|
|
244
|
-
void loadPage(api, ws, path, d.hash, d.type, p);
|
|
423
|
+
const first = Math.max(0, Math.min(Math.floor(startRow / pageSize), pageCount - 1));
|
|
424
|
+
const last = Math.max(first, Math.min(Math.ceil(endRow / pageSize) - 1, pageCount - 1));
|
|
425
|
+
const w = { hash: d.hash, first, last };
|
|
426
|
+
windows.set(datasetKey(ws, path), w);
|
|
427
|
+
const kept = pruneRetainedPages(d.mode.pages, first, last, MAX_RETAINED_PAGES);
|
|
428
|
+
const loading = wantedMissing(w, { ...d.mode, pages: kept });
|
|
429
|
+
if (kept !== d.mode.pages || !sameList(loading, d.mode.loading))
|
|
430
|
+
setMode(ws, path, d.hash, { ...d.mode, pages: kept, loading });
|
|
431
|
+
pump(ws, path);
|
|
245
432
|
},
|
|
246
433
|
async reload(ws, path) {
|
|
247
434
|
const d = current(ws, path);
|
|
248
435
|
if (d !== undefined && d.hash !== null) {
|
|
249
436
|
// Drop the whole-value in-flight key so the reload fetches afresh.
|
|
250
|
-
|
|
437
|
+
wholeInflight.delete(keyOf(ws, path, d.hash, 'whole'));
|
|
251
438
|
}
|
|
252
439
|
await loader.tick(ws, path, true);
|
|
253
440
|
},
|
|
@@ -288,8 +475,14 @@ export function createDatasetLoader(deps) {
|
|
|
288
475
|
return (await api.datasetGet(ws, treePathOf(path))).data;
|
|
289
476
|
},
|
|
290
477
|
reset() {
|
|
291
|
-
|
|
478
|
+
wholeInflight.clear();
|
|
292
479
|
cache.clear();
|
|
480
|
+
windows.clear();
|
|
481
|
+
inflight.clear();
|
|
482
|
+
retryAt.clear();
|
|
483
|
+
for (const { timer } of retryPump.values())
|
|
484
|
+
clearTimeout(timer);
|
|
485
|
+
retryPump.clear();
|
|
293
486
|
},
|
|
294
487
|
};
|
|
295
488
|
return loader;
|