@oceanum/oceanumlab 4.8.0 → 4.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/StoredNotebooks.js +31 -39
- package/lib/auth/tokens.d.ts +0 -6
- package/lib/share/fromDatamesh.d.ts +82 -0
- package/lib/share/fromDatamesh.js +379 -0
- package/lib/share/notebook.d.ts +6 -12
- package/lib/share/notebook.js +17 -37
- package/lib/share/plugin.js +58 -2
- package/package.json +1 -1
- package/schema/datamesh-connect.json +1 -1
- package/src/StoredNotebooks.tsx +58 -77
- package/src/__tests__/OceanumPanel.spec.tsx +12 -17
- package/src/__tests__/StoredNotebooks.spec.tsx +80 -87
- package/src/__tests__/fromDatamesh.spec.ts +301 -0
- package/src/__tests__/shareNotebook.spec.ts +32 -89
- package/src/__tests__/sharePlugin.spec.ts +57 -2
- package/src/auth/tokens.ts +0 -6
- package/src/share/fromDatamesh.ts +477 -0
- package/src/share/notebook.ts +21 -52
- package/src/share/plugin.ts +69 -1
- package/style/index.css +0 -7
package/lib/StoredNotebooks.js
CHANGED
|
@@ -1,28 +1,32 @@
|
|
|
1
1
|
import * as React from 'react';
|
|
2
|
-
import { SpecStoreClient } from './share/client';
|
|
2
|
+
import { SpecStoreClient, SpecStoreError } from './share/client';
|
|
3
3
|
import { notebooksChanged } from './share/events';
|
|
4
|
-
import {
|
|
4
|
+
import { demoItemsFromSummaries, NOTEBOOK_DEMO_TYPE, partitionSummaries } from './share/notebook';
|
|
5
5
|
import { SPEC_ID_ATTRIBUTE } from './share/plugin';
|
|
6
6
|
/** Example rows carry their record id here, not in SPEC_ID_ATTRIBUTE: see Examples. */
|
|
7
7
|
export const EXAMPLE_ID_ATTRIBUTE = 'data-example-id';
|
|
8
8
|
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
9
|
+
* List the Notebook Demo specs, each one an example notebook. Only org oceanum may
|
|
10
|
+
* create them (spec store 0.6.2), so every one listed is Oceanum's. Never rejects: a
|
|
11
|
+
* failure here is shown in the Examples section and leaves the rest of the tab alone.
|
|
11
12
|
*/
|
|
12
|
-
async function loadExamples(auth, specsUrl
|
|
13
|
-
if (!id) {
|
|
14
|
-
return { state: 'none' };
|
|
15
|
-
}
|
|
13
|
+
async function loadExamples(auth, specsUrl) {
|
|
16
14
|
try {
|
|
17
15
|
const client = new SpecStoreClient({
|
|
18
16
|
specsUrl,
|
|
19
17
|
getAccessToken: () => auth.getAccessToken(),
|
|
20
|
-
specType:
|
|
18
|
+
specType: NOTEBOOK_DEMO_TYPE
|
|
21
19
|
});
|
|
22
|
-
|
|
23
|
-
|
|
20
|
+
return {
|
|
21
|
+
state: 'ready',
|
|
22
|
+
items: demoItemsFromSummaries(await client.list())
|
|
23
|
+
};
|
|
24
24
|
}
|
|
25
25
|
catch (error) {
|
|
26
|
+
// A store with no Notebook Demo type has no examples, which is not an error.
|
|
27
|
+
if (error instanceof SpecStoreError && error.kind === 'not-found') {
|
|
28
|
+
return { state: 'none' };
|
|
29
|
+
}
|
|
26
30
|
return {
|
|
27
31
|
state: 'error',
|
|
28
32
|
message: error instanceof Error ? error.message : 'Could not reach Oceanum.io.'
|
|
@@ -30,18 +34,17 @@ async function loadExamples(auth, specsUrl, id) {
|
|
|
30
34
|
}
|
|
31
35
|
}
|
|
32
36
|
/**
|
|
33
|
-
* The
|
|
34
|
-
* stored-notebook row class: the share plugin's row menu
|
|
35
|
-
* is bound to those, and none of it applies to
|
|
37
|
+
* The examples: Oceanum's Notebook Demo specs. Their rows deliberately leave out
|
|
38
|
+
* SPEC_ID_ATTRIBUTE and the stored-notebook row class: the share plugin's row menu
|
|
39
|
+
* (Open, Rename, Share, Delete) is bound to those, and none of it applies to
|
|
40
|
+
* Oceanum's own example records.
|
|
36
41
|
*/
|
|
37
42
|
function ExampleSections({ examples, onOpen, show, onShowChange }) {
|
|
38
43
|
// Not yet known, or hidden with no switch to bring them back: nothing to show.
|
|
39
44
|
if (examples.state === 'none' || show === null || (!show && !onShowChange)) {
|
|
40
45
|
return null;
|
|
41
46
|
}
|
|
42
|
-
const count = examples.state === 'ready'
|
|
43
|
-
? examples.sections.reduce((total, section) => total + section.items.length, 0)
|
|
44
|
-
: null;
|
|
47
|
+
const count = examples.state === 'ready' ? examples.items.length : null;
|
|
45
48
|
if (count === 0) {
|
|
46
49
|
return null;
|
|
47
50
|
}
|
|
@@ -64,18 +67,15 @@ function ExampleSections({ examples, onOpen, show, onShowChange }) {
|
|
|
64
67
|
}
|
|
65
68
|
return (React.createElement("div", { className: "oceanum-notebooks-section" },
|
|
66
69
|
heading,
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
// A record may list the same example twice; the id alone is no key.
|
|
77
|
-
React.createElement("li", { key: `${position}:${item.id}` }, onOpen ? (React.createElement("button", { type: "button", className: "oceanum-notebooks-example", onClick: () => onOpen(item), ...attributes }, name)) : (React.createElement("div", { className: "oceanum-notebooks-example oceanum-notebooks-item-static", ...attributes }, name))));
|
|
78
|
-
})))))));
|
|
70
|
+
React.createElement("ul", { className: "oceanum-notebooks-list" }, examples.items.map(item => {
|
|
71
|
+
const attributes = {
|
|
72
|
+
[EXAMPLE_ID_ATTRIBUTE]: item.id,
|
|
73
|
+
title: item.summary
|
|
74
|
+
};
|
|
75
|
+
// Names come from the spec store: React escapes them, never set as HTML.
|
|
76
|
+
const name = (React.createElement("span", { className: "oceanum-notebooks-name" }, item.title));
|
|
77
|
+
return (React.createElement("li", { key: item.id }, onOpen ? (React.createElement("button", { type: "button", className: "oceanum-notebooks-example", onClick: () => onOpen(item), ...attributes }, name)) : (React.createElement("div", { className: "oceanum-notebooks-example oceanum-notebooks-item-static", ...attributes }, name))));
|
|
78
|
+
}))));
|
|
79
79
|
}
|
|
80
80
|
function Section({ title, items, empty, onOpen }) {
|
|
81
81
|
// Each row carries its record id so the share plugin's context menu (Open, Rename,
|
|
@@ -127,17 +127,14 @@ export function StoredNotebooks({ auth, onOpen, onOpenExample, onUpload, showExa
|
|
|
127
127
|
}
|
|
128
128
|
setLoad({ state: 'loading' });
|
|
129
129
|
void (async () => {
|
|
130
|
-
var _a, _b;
|
|
131
130
|
try {
|
|
132
131
|
const client = new SpecStoreClient({
|
|
133
132
|
specsUrl: specs,
|
|
134
133
|
getAccessToken: () => auth.getAccessToken()
|
|
135
134
|
});
|
|
136
|
-
// Only the record the deployment names: every signed-in user can create
|
|
137
|
-
// notebook-demo records, so the examples never come from a listing.
|
|
138
135
|
const [items, examples] = await Promise.all([
|
|
139
136
|
client.list(),
|
|
140
|
-
loadExamples(auth, specs
|
|
137
|
+
loadExamples(auth, specs)
|
|
141
138
|
]);
|
|
142
139
|
if (!cancelled) {
|
|
143
140
|
setLoad({ state: 'ready', items, examples });
|
|
@@ -187,15 +184,10 @@ export function StoredNotebooks({ auth, onOpen, onOpenExample, onUpload, showExa
|
|
|
187
184
|
return React.createElement("div", { className: "oceanum-text-error" }, load.message);
|
|
188
185
|
}
|
|
189
186
|
const { mine, shared } = partitionSummaries(load.items, user.email);
|
|
190
|
-
// A public example is readable by everyone, so the listing returns it as shared with
|
|
191
|
-
// every signed-in user; it is shown under Examples instead.
|
|
192
|
-
const exampleIds = new Set(load.examples.state === 'ready'
|
|
193
|
-
? load.examples.sections.flatMap(section => section.items.map(item => item.id))
|
|
194
|
-
: []);
|
|
195
187
|
return (React.createElement("div", { className: "oceanum-notebooks" },
|
|
196
188
|
onUpload && ((_a = auth.urls) === null || _a === void 0 ? void 0 : _a.specs) ? (React.createElement("div", { className: "oceanum-notebooks-actions" },
|
|
197
189
|
React.createElement("button", { type: "button", className: "oceanum-notebooks-upload", title: "Upload a notebook from this computer, to save on Oceanum.io", onClick: onUpload }, "Upload notebook\u2026"))) : null,
|
|
198
190
|
React.createElement(Section, { title: "My notebooks", items: mine, empty: "You have not saved any notebooks yet.", onOpen: onOpen }),
|
|
199
|
-
React.createElement(Section, { title: "Shared with me", items: shared
|
|
191
|
+
React.createElement(Section, { title: "Shared with me", items: shared, empty: "No notebooks have been shared with you.", onOpen: onOpen }),
|
|
200
192
|
React.createElement(ExampleSections, { examples: load.examples, onOpen: onOpenExample, show: showExamples, onShowChange: onShowExamplesChange })));
|
|
201
193
|
}
|
package/lib/auth/tokens.d.ts
CHANGED
|
@@ -31,12 +31,6 @@ export interface IOceanumEnvironment {
|
|
|
31
31
|
* absent) keeps JupyterLab's file management alongside the Oceanum commands.
|
|
32
32
|
*/
|
|
33
33
|
readonly fileManagement?: 'local' | 'oceanum';
|
|
34
|
-
/**
|
|
35
|
-
* The spec store id of this deployment's Notebook Demo record, the curated list the
|
|
36
|
-
* Notebooks tab shows as Examples. Absent where a deployment has none, and never set
|
|
37
|
-
* by a JupyterLab server's sign-in.
|
|
38
|
-
*/
|
|
39
|
-
readonly notebookDemo?: string;
|
|
40
34
|
}
|
|
41
35
|
/** Service base URLs for an environment, without trailing slashes. */
|
|
42
36
|
export interface IOceanumServiceUrls {
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Notebooks generated from a Datamesh UI workspace or query.
|
|
3
|
+
*
|
|
4
|
+
* The Datamesh UI's "Open in notebook" button opens this app with the workspace in the
|
|
5
|
+
* address fragment, `#oceanum-datamesh=<base64url JSON>`. The fragment is never sent
|
|
6
|
+
* to a server, so the queries stay out of request logs and are not bound by URL
|
|
7
|
+
* length limits on the way in.
|
|
8
|
+
*
|
|
9
|
+
* Anyone can craft such a link, so the payload is treated as data only: it is
|
|
10
|
+
* validated, every value reaches the notebook as a Python literal, and the notebook is
|
|
11
|
+
* built from fixed templates, opened untrusted and never run automatically.
|
|
12
|
+
*/
|
|
13
|
+
import type { INotebookContent } from '@jupyterlab/nbformat';
|
|
14
|
+
/** Fragment key carrying a Datamesh workspace. */
|
|
15
|
+
export declare const FRAGMENT_KEY = "oceanum-datamesh";
|
|
16
|
+
/** Payload version this module reads. */
|
|
17
|
+
export declare const PAYLOAD_VERSION = 1;
|
|
18
|
+
/** Most queries one link may carry. */
|
|
19
|
+
export declare const MAX_ITEMS = 100;
|
|
20
|
+
/** Longest encoded payload accepted (characters of base64url). */
|
|
21
|
+
export declare const MAX_ENCODED_LENGTH: number;
|
|
22
|
+
/**
|
|
23
|
+
* Deepest nesting of arrays and objects accepted. Real queries, GeoJSON
|
|
24
|
+
* multipolygon geofilters included, nest about 8 deep.
|
|
25
|
+
*/
|
|
26
|
+
export declare const MAX_DEPTH = 32;
|
|
27
|
+
/** One query: an OceanQL spec plus the Datamesh UI's own `label`. */
|
|
28
|
+
export interface IDatameshQuery {
|
|
29
|
+
datasource: string;
|
|
30
|
+
label?: string | null;
|
|
31
|
+
description?: string | null;
|
|
32
|
+
[key: string]: unknown;
|
|
33
|
+
}
|
|
34
|
+
/** What the fragment carries. */
|
|
35
|
+
export interface IDatameshNotebookRequest {
|
|
36
|
+
v: typeof PAYLOAD_VERSION;
|
|
37
|
+
/** Workspace name, used for the notebook name and heading. */
|
|
38
|
+
name?: string;
|
|
39
|
+
items: IDatameshQuery[];
|
|
40
|
+
}
|
|
41
|
+
/** UTF-8 JSON to base64url, without padding. */
|
|
42
|
+
export declare function encodePayload(request: IDatameshNotebookRequest): string;
|
|
43
|
+
/**
|
|
44
|
+
* Read and validate a payload. Throws an `Error` whose message can be shown to the
|
|
45
|
+
* user; it never quotes the payload.
|
|
46
|
+
*/
|
|
47
|
+
export declare function decodePayload(encoded: string): IDatameshNotebookRequest;
|
|
48
|
+
/**
|
|
49
|
+
* The payload in a location hash (`#oceanum-datamesh=...`), or null when the hash
|
|
50
|
+
* carries none. Other parts of the hash are ignored.
|
|
51
|
+
*/
|
|
52
|
+
export declare function payloadFromHash(hash: string): string | null;
|
|
53
|
+
/**
|
|
54
|
+
* The hash with the payload removed (`''` when nothing else is left). Every other
|
|
55
|
+
* part is kept exactly as it was.
|
|
56
|
+
*/
|
|
57
|
+
export declare function hashWithoutPayload(hash: string): string;
|
|
58
|
+
/**
|
|
59
|
+
* A Python string literal. JSON's escaping is valid Python and escapes C0 controls;
|
|
60
|
+
* the C1 controls and U+2028/U+2029 it leaves raw are escaped too, so the generated
|
|
61
|
+
* source has no character a kernel could treat as a line break.
|
|
62
|
+
*/
|
|
63
|
+
export declare function pyString(text: string): string;
|
|
64
|
+
/**
|
|
65
|
+
* A Python literal for a JSON value, wrapped over indented lines when it does not
|
|
66
|
+
* fit on one. Strings and keys go through `pyString`, so no value can end the
|
|
67
|
+
* literal early or break the line.
|
|
68
|
+
*/
|
|
69
|
+
export declare function pyLiteral(value: unknown, indent?: number): string;
|
|
70
|
+
/** A Python identifier for `text`, not in `taken`; adds it to `taken`. */
|
|
71
|
+
export declare function pyIdentifier(text: string, taken: Set<string>): string;
|
|
72
|
+
/** The OceanQL query to send: only fields oceanum-python's `Query` accepts. */
|
|
73
|
+
export declare function queryFields(item: IDatameshQuery): Record<string, unknown>;
|
|
74
|
+
/** The code cell that loads one query into a variable and shows it. */
|
|
75
|
+
export declare function queryCellSource(item: IDatameshQuery, variable: string): string;
|
|
76
|
+
/** The notebook name for a request: the workspace name, or the single query's. */
|
|
77
|
+
export declare function notebookName(request: IDatameshNotebookRequest): string;
|
|
78
|
+
/**
|
|
79
|
+
* A new notebook that loads every query in `request`. It carries no `oceanum` link and
|
|
80
|
+
* no trust, so saving it creates a record of the user's own.
|
|
81
|
+
*/
|
|
82
|
+
export declare function notebookFromDatamesh(request: IDatameshNotebookRequest): INotebookContent;
|
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
/** Fragment key carrying a Datamesh workspace. */
|
|
2
|
+
export const FRAGMENT_KEY = 'oceanum-datamesh';
|
|
3
|
+
/** Payload version this module reads. */
|
|
4
|
+
export const PAYLOAD_VERSION = 1;
|
|
5
|
+
/** Most queries one link may carry. */
|
|
6
|
+
export const MAX_ITEMS = 100;
|
|
7
|
+
/** Longest encoded payload accepted (characters of base64url). */
|
|
8
|
+
export const MAX_ENCODED_LENGTH = 4 * 1024 * 1024;
|
|
9
|
+
/**
|
|
10
|
+
* Deepest nesting of arrays and objects accepted. Real queries, GeoJSON
|
|
11
|
+
* multipolygon geofilters included, nest about 8 deep.
|
|
12
|
+
*/
|
|
13
|
+
export const MAX_DEPTH = 32;
|
|
14
|
+
/** Nesting depth of a JSON value (a scalar is 0), without recursion. */
|
|
15
|
+
function depthOf(value) {
|
|
16
|
+
let max = 0;
|
|
17
|
+
const stack = [[value, 0]];
|
|
18
|
+
while (stack.length) {
|
|
19
|
+
const [v, d] = stack.pop();
|
|
20
|
+
if (v !== null && typeof v === 'object') {
|
|
21
|
+
max = Math.max(max, d + 1);
|
|
22
|
+
if (max > MAX_DEPTH) {
|
|
23
|
+
return max;
|
|
24
|
+
}
|
|
25
|
+
for (const child of Object.values(v)) {
|
|
26
|
+
stack.push([child, d + 1]);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return max;
|
|
31
|
+
}
|
|
32
|
+
/** The fields of oceanum-python's `Query`. Anything else would make it warn. */
|
|
33
|
+
const QUERY_FIELDS = [
|
|
34
|
+
'datasource',
|
|
35
|
+
'parameters',
|
|
36
|
+
'description',
|
|
37
|
+
'variables',
|
|
38
|
+
'timefilter',
|
|
39
|
+
'geofilter',
|
|
40
|
+
'levelfilter',
|
|
41
|
+
'coordfilter',
|
|
42
|
+
'crs',
|
|
43
|
+
'aggregate',
|
|
44
|
+
'functions',
|
|
45
|
+
'limit'
|
|
46
|
+
];
|
|
47
|
+
/** Fields that make a query select something rather than the whole datasource. */
|
|
48
|
+
const FILTER_FIELDS = QUERY_FIELDS.filter(f => f !== 'datasource' && f !== 'description');
|
|
49
|
+
function isObject(value) {
|
|
50
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
51
|
+
}
|
|
52
|
+
/** UTF-8 JSON to base64url, without padding. */
|
|
53
|
+
export function encodePayload(request) {
|
|
54
|
+
const bytes = new TextEncoder().encode(JSON.stringify(request));
|
|
55
|
+
let binary = '';
|
|
56
|
+
for (const b of bytes) {
|
|
57
|
+
binary += String.fromCharCode(b);
|
|
58
|
+
}
|
|
59
|
+
return btoa(binary)
|
|
60
|
+
.replace(/\+/g, '-')
|
|
61
|
+
.replace(/\//g, '_')
|
|
62
|
+
.replace(/=+$/, '');
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Read and validate a payload. Throws an `Error` whose message can be shown to the
|
|
66
|
+
* user; it never quotes the payload.
|
|
67
|
+
*/
|
|
68
|
+
export function decodePayload(encoded) {
|
|
69
|
+
if (encoded.length > MAX_ENCODED_LENGTH) {
|
|
70
|
+
throw new Error('The Datamesh link is too large to open.');
|
|
71
|
+
}
|
|
72
|
+
let parsed;
|
|
73
|
+
try {
|
|
74
|
+
const b64 = encoded.replace(/-/g, '+').replace(/_/g, '/');
|
|
75
|
+
const binary = atob(b64);
|
|
76
|
+
const bytes = Uint8Array.from(binary, c => c.charCodeAt(0));
|
|
77
|
+
parsed = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes));
|
|
78
|
+
}
|
|
79
|
+
catch (_a) {
|
|
80
|
+
throw new Error('The Datamesh link is not valid.');
|
|
81
|
+
}
|
|
82
|
+
if (!isObject(parsed) || parsed.v !== PAYLOAD_VERSION) {
|
|
83
|
+
throw new Error('The Datamesh link is not valid, or comes from a newer Datamesh UI.');
|
|
84
|
+
}
|
|
85
|
+
const { name, items } = parsed;
|
|
86
|
+
// null is accepted wherever a string is optional: the UI copies datasource
|
|
87
|
+
// properties that may be null.
|
|
88
|
+
const optionalText = (value) => value === undefined || value === null || typeof value === 'string';
|
|
89
|
+
if (!optionalText(name)) {
|
|
90
|
+
throw new Error('The Datamesh link is not valid.');
|
|
91
|
+
}
|
|
92
|
+
if (!Array.isArray(items) || items.length === 0) {
|
|
93
|
+
throw new Error('The Datamesh link holds no queries.');
|
|
94
|
+
}
|
|
95
|
+
if (items.length > MAX_ITEMS) {
|
|
96
|
+
throw new Error(`The Datamesh link holds more than ${MAX_ITEMS} queries, which is too many to open.`);
|
|
97
|
+
}
|
|
98
|
+
if (depthOf(items) > MAX_DEPTH) {
|
|
99
|
+
throw new Error('The Datamesh link holds a query that is not valid.');
|
|
100
|
+
}
|
|
101
|
+
for (const item of items) {
|
|
102
|
+
if (!isObject(item) ||
|
|
103
|
+
typeof item.datasource !== 'string' ||
|
|
104
|
+
item.datasource.trim() === '' ||
|
|
105
|
+
!optionalText(item.label) ||
|
|
106
|
+
!optionalText(item.description)) {
|
|
107
|
+
throw new Error('The Datamesh link holds a query that is not valid.');
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return {
|
|
111
|
+
v: PAYLOAD_VERSION,
|
|
112
|
+
...(typeof name === 'string' ? { name } : {}),
|
|
113
|
+
items: items
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
const PAYLOAD_PREFIX = `${FRAGMENT_KEY}=`;
|
|
117
|
+
/** The hash's `&`-separated parts, without the leading `#`. */
|
|
118
|
+
function hashParts(hash) {
|
|
119
|
+
const body = hash.replace(/^#/, '');
|
|
120
|
+
return body === '' ? [] : body.split('&');
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* The payload in a location hash (`#oceanum-datamesh=...`), or null when the hash
|
|
124
|
+
* carries none. Other parts of the hash are ignored.
|
|
125
|
+
*/
|
|
126
|
+
export function payloadFromHash(hash) {
|
|
127
|
+
const part = hashParts(hash).find(p => p.startsWith(PAYLOAD_PREFIX));
|
|
128
|
+
return part === undefined ? null : part.slice(PAYLOAD_PREFIX.length);
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* The hash with the payload removed (`''` when nothing else is left). Every other
|
|
132
|
+
* part is kept exactly as it was.
|
|
133
|
+
*/
|
|
134
|
+
export function hashWithoutPayload(hash) {
|
|
135
|
+
const rest = hashParts(hash).filter(p => !p.startsWith(PAYLOAD_PREFIX));
|
|
136
|
+
return rest.length ? `#${rest.join('&')}` : '';
|
|
137
|
+
}
|
|
138
|
+
const INLINE_WIDTH = 72;
|
|
139
|
+
/** Characters JSON leaves raw that Python's `splitlines()` still breaks on. */
|
|
140
|
+
const RAW_BREAKS = new RegExp(`[\\x7f-\\x9f${String.fromCharCode(0x2028, 0x2029)}]`, 'g');
|
|
141
|
+
/**
|
|
142
|
+
* A Python string literal. JSON's escaping is valid Python and escapes C0 controls;
|
|
143
|
+
* the C1 controls and U+2028/U+2029 it leaves raw are escaped too, so the generated
|
|
144
|
+
* source has no character a kernel could treat as a line break.
|
|
145
|
+
*/
|
|
146
|
+
export function pyString(text) {
|
|
147
|
+
return JSON.stringify(text).replace(RAW_BREAKS, c => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
|
148
|
+
}
|
|
149
|
+
/** A Python literal for a scalar, or null for an array or object. */
|
|
150
|
+
function pyScalar(value) {
|
|
151
|
+
if (value === null || value === undefined) {
|
|
152
|
+
return 'None';
|
|
153
|
+
}
|
|
154
|
+
if (typeof value === 'boolean') {
|
|
155
|
+
return value ? 'True' : 'False';
|
|
156
|
+
}
|
|
157
|
+
if (typeof value === 'number') {
|
|
158
|
+
// Parsed JSON has no NaN or Infinity; guard anyway rather than emit bad code.
|
|
159
|
+
return Number.isFinite(value) ? JSON.stringify(value) : 'None';
|
|
160
|
+
}
|
|
161
|
+
if (typeof value === 'string') {
|
|
162
|
+
return pyString(value);
|
|
163
|
+
}
|
|
164
|
+
return Array.isArray(value) || isObject(value) ? null : 'None';
|
|
165
|
+
}
|
|
166
|
+
function definedEntries(value) {
|
|
167
|
+
return Object.entries(value).filter(([, v]) => v !== undefined);
|
|
168
|
+
}
|
|
169
|
+
/** A Python literal on one line, rendering each node once. */
|
|
170
|
+
function pyInline(value) {
|
|
171
|
+
const scalar = pyScalar(value);
|
|
172
|
+
if (scalar !== null) {
|
|
173
|
+
return scalar;
|
|
174
|
+
}
|
|
175
|
+
if (Array.isArray(value)) {
|
|
176
|
+
return `[${value.map(pyInline).join(', ')}]`;
|
|
177
|
+
}
|
|
178
|
+
return `{${definedEntries(value)
|
|
179
|
+
.map(([k, v]) => `${pyString(k)}: ${pyInline(v)}`)
|
|
180
|
+
.join(', ')}}`;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* A Python literal for a JSON value, wrapped over indented lines when it does not
|
|
184
|
+
* fit on one. Strings and keys go through `pyString`, so no value can end the
|
|
185
|
+
* literal early or break the line.
|
|
186
|
+
*/
|
|
187
|
+
export function pyLiteral(value, indent = 0) {
|
|
188
|
+
const scalar = pyScalar(value);
|
|
189
|
+
if (scalar !== null) {
|
|
190
|
+
return scalar;
|
|
191
|
+
}
|
|
192
|
+
const pad = ' '.repeat(indent + 4);
|
|
193
|
+
const end = ' '.repeat(indent);
|
|
194
|
+
const inline = pyInline(value);
|
|
195
|
+
if (Array.isArray(value)) {
|
|
196
|
+
if (inline.length + indent <= INLINE_WIDTH || value.length === 0) {
|
|
197
|
+
return inline;
|
|
198
|
+
}
|
|
199
|
+
const lines = value.map(v => pad + pyLiteral(v, indent + 4));
|
|
200
|
+
return `[\n${lines.join(',\n')},\n${end}]`;
|
|
201
|
+
}
|
|
202
|
+
if (isObject(value)) {
|
|
203
|
+
const entries = definedEntries(value);
|
|
204
|
+
if (inline.length + indent <= INLINE_WIDTH || entries.length === 0) {
|
|
205
|
+
return inline;
|
|
206
|
+
}
|
|
207
|
+
const lines = entries.map(([k, v]) => `${pad}${pyString(k)}: ${pyLiteral(v, indent + 4)}`);
|
|
208
|
+
return `{\n${lines.join(',\n')},\n${end}}`;
|
|
209
|
+
}
|
|
210
|
+
return 'None';
|
|
211
|
+
}
|
|
212
|
+
const PY_KEYWORDS = new Set([
|
|
213
|
+
'False',
|
|
214
|
+
'None',
|
|
215
|
+
'True',
|
|
216
|
+
'and',
|
|
217
|
+
'as',
|
|
218
|
+
'assert',
|
|
219
|
+
'async',
|
|
220
|
+
'await',
|
|
221
|
+
'break',
|
|
222
|
+
'class',
|
|
223
|
+
'continue',
|
|
224
|
+
'def',
|
|
225
|
+
'del',
|
|
226
|
+
'elif',
|
|
227
|
+
'else',
|
|
228
|
+
'except',
|
|
229
|
+
'finally',
|
|
230
|
+
'for',
|
|
231
|
+
'from',
|
|
232
|
+
'global',
|
|
233
|
+
'if',
|
|
234
|
+
'import',
|
|
235
|
+
'in',
|
|
236
|
+
'is',
|
|
237
|
+
'lambda',
|
|
238
|
+
'nonlocal',
|
|
239
|
+
'not',
|
|
240
|
+
'or',
|
|
241
|
+
'pass',
|
|
242
|
+
'raise',
|
|
243
|
+
'return',
|
|
244
|
+
'try',
|
|
245
|
+
'while',
|
|
246
|
+
'with',
|
|
247
|
+
'yield',
|
|
248
|
+
// Names the generated code itself uses.
|
|
249
|
+
'datamesh',
|
|
250
|
+
'Connector'
|
|
251
|
+
]);
|
|
252
|
+
/** A Python identifier for `text`, not in `taken`; adds it to `taken`. */
|
|
253
|
+
export function pyIdentifier(text, taken) {
|
|
254
|
+
let base = text
|
|
255
|
+
.replace(/[^A-Za-z0-9_]+/g, '_')
|
|
256
|
+
.replace(/_+/g, '_')
|
|
257
|
+
.replace(/^_+|_+$/g, '')
|
|
258
|
+
.toLowerCase();
|
|
259
|
+
if (base === '') {
|
|
260
|
+
base = 'data';
|
|
261
|
+
}
|
|
262
|
+
if (/^[0-9]/.test(base)) {
|
|
263
|
+
base = `ds_${base}`;
|
|
264
|
+
}
|
|
265
|
+
if (PY_KEYWORDS.has(base)) {
|
|
266
|
+
base = `${base}_`;
|
|
267
|
+
}
|
|
268
|
+
let name = base;
|
|
269
|
+
for (let i = 2; taken.has(name); i++) {
|
|
270
|
+
name = `${base}_${i}`;
|
|
271
|
+
}
|
|
272
|
+
taken.add(name);
|
|
273
|
+
return name;
|
|
274
|
+
}
|
|
275
|
+
/**
|
|
276
|
+
* Every C0 and C1 control character, plus U+2028 and U+2029. This is a superset of
|
|
277
|
+
* what Python's `str.splitlines()` breaks on (LF, CR, VT, FF, FS, GS, RS, NEL and the
|
|
278
|
+
* two separators), which IPython's input transformer uses to split a cell into lines:
|
|
279
|
+
* left in a `#` comment, any of them would end the comment and run what follows.
|
|
280
|
+
*/
|
|
281
|
+
const LINE_BREAKS = new RegExp(`[\\x00-\\x1f\\x7f-\\x9f${String.fromCharCode(0x2028, 0x2029)}]+`, 'g');
|
|
282
|
+
/** One line of text safe to put after a `#` comment marker. */
|
|
283
|
+
function commentText(text) {
|
|
284
|
+
return text.replace(LINE_BREAKS, ' ').trim();
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Text safe to put in a Markdown heading: no HTML, no Markdown syntax, and no
|
|
288
|
+
* autolink (`:`, `/` and `.` are escaped too). `&`, `<` and `>` become entities;
|
|
289
|
+
* every other ASCII punctuation character is backslash-escaped except `;` (part
|
|
290
|
+
* of the entities) and `@`: JupyterLab's math handling keeps `@@0@@` as a
|
|
291
|
+
* placeholder, and an escaped `\@\@0\@\@` slips past it and breaks rendering.
|
|
292
|
+
*/
|
|
293
|
+
function markdownText(text) {
|
|
294
|
+
return commentText(text)
|
|
295
|
+
.replace(/&/g, '&')
|
|
296
|
+
.replace(/</g, '<')
|
|
297
|
+
.replace(/>/g, '>')
|
|
298
|
+
.replace(/([!-%'-/:=?[-`{-~])/g, '\\$1');
|
|
299
|
+
}
|
|
300
|
+
/** The OceanQL query to send: only fields oceanum-python's `Query` accepts. */
|
|
301
|
+
export function queryFields(item) {
|
|
302
|
+
const query = {};
|
|
303
|
+
for (const field of QUERY_FIELDS) {
|
|
304
|
+
if (item[field] !== undefined && item[field] !== null) {
|
|
305
|
+
query[field] = item[field];
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
return query;
|
|
309
|
+
}
|
|
310
|
+
/** Whether a query selects part of its datasource (else it loads the whole one). */
|
|
311
|
+
function isFiltered(query) {
|
|
312
|
+
return FILTER_FIELDS.some(f => {
|
|
313
|
+
const v = query[f];
|
|
314
|
+
return !(Array.isArray(v) && v.length === 0) && v !== undefined;
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
/** The code cell that loads one query into a variable and shows it. */
|
|
318
|
+
export function queryCellSource(item, variable) {
|
|
319
|
+
var _a, _b;
|
|
320
|
+
const query = queryFields(item);
|
|
321
|
+
const lines = [];
|
|
322
|
+
const title = commentText((_b = (_a = item.description) !== null && _a !== void 0 ? _a : item.label) !== null && _b !== void 0 ? _b : '');
|
|
323
|
+
if (title) {
|
|
324
|
+
lines.push(`# ${title}`);
|
|
325
|
+
}
|
|
326
|
+
if (isFiltered(query)) {
|
|
327
|
+
lines.push(`${variable} = datamesh.query(${pyLiteral(query)})`);
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
// No filter: open the datasource lazily rather than download all of it.
|
|
331
|
+
lines.push(`${variable} = datamesh.load_datasource(${pyLiteral(item.datasource)})`);
|
|
332
|
+
}
|
|
333
|
+
lines.push(variable);
|
|
334
|
+
return lines.join('\n');
|
|
335
|
+
}
|
|
336
|
+
function codeCell(source) {
|
|
337
|
+
return {
|
|
338
|
+
cell_type: 'code',
|
|
339
|
+
execution_count: null,
|
|
340
|
+
metadata: {},
|
|
341
|
+
outputs: [],
|
|
342
|
+
source
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
/** The notebook name for a request: the workspace name, or the single query's. */
|
|
346
|
+
export function notebookName(request) {
|
|
347
|
+
var _a, _b;
|
|
348
|
+
const only = request.items.length === 1 ? request.items[0] : null;
|
|
349
|
+
const name = ((_a = request.name) === null || _a === void 0 ? void 0 : _a.trim()) ||
|
|
350
|
+
((_b = only === null || only === void 0 ? void 0 : only.label) === null || _b === void 0 ? void 0 : _b.trim()) ||
|
|
351
|
+
(only === null || only === void 0 ? void 0 : only.datasource.trim()) ||
|
|
352
|
+
'Datamesh workspace';
|
|
353
|
+
return commentText(name);
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* A new notebook that loads every query in `request`. It carries no `oceanum` link and
|
|
357
|
+
* no trust, so saving it creates a record of the user's own.
|
|
358
|
+
*/
|
|
359
|
+
export function notebookFromDatamesh(request) {
|
|
360
|
+
const taken = new Set();
|
|
361
|
+
const cells = [
|
|
362
|
+
{
|
|
363
|
+
cell_type: 'markdown',
|
|
364
|
+
metadata: {},
|
|
365
|
+
source: `# ${markdownText(notebookName(request))}\n\n` +
|
|
366
|
+
'Opened from the Oceanum Datamesh UI. Run the cells to load the data.'
|
|
367
|
+
},
|
|
368
|
+
codeCell('from oceanum.datamesh import Connector\n\ndatamesh = Connector()'),
|
|
369
|
+
...request.items.map(item => codeCell(queryCellSource(item, pyIdentifier(item.label || item.datasource, taken))))
|
|
370
|
+
];
|
|
371
|
+
return {
|
|
372
|
+
nbformat: 4,
|
|
373
|
+
nbformat_minor: 5,
|
|
374
|
+
// No kernelspec: the kernel is `python` in JupyterLite and `python3` in
|
|
375
|
+
// JupyterLab, and the language alone lets either pick its Python kernel.
|
|
376
|
+
metadata: { language_info: { name: 'python' } },
|
|
377
|
+
cells
|
|
378
|
+
};
|
|
379
|
+
}
|
package/lib/share/notebook.d.ts
CHANGED
|
@@ -141,25 +141,19 @@ export declare function partitionSummaries(items: readonly ISpecSummary[], email
|
|
|
141
141
|
mine: ISpecSummary[];
|
|
142
142
|
shared: ISpecSummary[];
|
|
143
143
|
};
|
|
144
|
-
/** The
|
|
145
|
-
export declare const
|
|
146
|
-
/** One example
|
|
144
|
+
/** The spec store type of the Notebook Demo specs, which are the Examples. */
|
|
145
|
+
export declare const NOTEBOOK_DEMO_TYPE = "notebook-demo";
|
|
146
|
+
/** One example: a Notebook Demo spec, whose body is the example notebook. */
|
|
147
147
|
export interface INotebookDemoItem {
|
|
148
148
|
id: string;
|
|
149
149
|
title: string;
|
|
150
150
|
summary?: string;
|
|
151
151
|
}
|
|
152
|
-
/** A titled group of examples. */
|
|
153
|
-
export interface INotebookDemoSection {
|
|
154
|
-
title: string;
|
|
155
|
-
items: INotebookDemoItem[];
|
|
156
|
-
}
|
|
157
152
|
/**
|
|
158
|
-
* The
|
|
159
|
-
*
|
|
160
|
-
* A spec of another kind or version gives no sections, with a warning.
|
|
153
|
+
* The examples in a listing of Notebook Demo specs, ordered by name as people number
|
|
154
|
+
* them ("2 …" before "10 …").
|
|
161
155
|
*/
|
|
162
|
-
export declare function
|
|
156
|
+
export declare function demoItemsFromSummaries(summaries: readonly ISpecSummary[]): INotebookDemoItem[];
|
|
163
157
|
/** Parse a spec store timestamp, which is UTC but usually has no zone designator. */
|
|
164
158
|
export declare function parseTimestamp(value: string): Date | null;
|
|
165
159
|
/**
|