@oceanmetrics/engine 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/LICENSE +21 -0
- package/README.md +194 -2
- package/dist/cells.d.ts +84 -0
- package/dist/cells.js +179 -0
- package/dist/chunks.d.ts +29 -0
- package/dist/chunks.js +49 -0
- package/dist/dataset.d.ts +72 -0
- package/dist/dataset.js +115 -0
- package/dist/engine.d.ts +72 -0
- package/dist/engine.js +154 -0
- package/dist/erddap.d.ts +117 -0
- package/dist/erddap.js +224 -0
- package/dist/extent.d.ts +64 -0
- package/dist/extent.js +151 -0
- package/dist/fetchRetry.d.ts +33 -0
- package/dist/fetchRetry.js +76 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +16 -0
- package/dist/lobes.d.ts +24 -0
- package/dist/lobes.js +55 -0
- package/dist/mask.d.ts +71 -0
- package/dist/mask.js +363 -0
- package/dist/preview.d.ts +121 -0
- package/dist/preview.js +205 -0
- package/dist/sql.d.ts +10 -0
- package/dist/sql.js +33 -0
- package/dist/sqlTemplates.d.ts +1 -0
- package/dist/sqlTemplates.js +8 -0
- package/dist/wkb.d.ts +5 -0
- package/dist/wkb.js +80 -0
- package/package.json +71 -4
- package/sql/last_step.sql +31 -0
- package/sql/points_tabledap.sql +21 -0
- package/sql/stats_categorical.sql +36 -0
- package/sql/stats_daily.sql +27 -0
- package/sql/stats_tabledap.sql +36 -0
- package/src/cells.ts +190 -0
- package/src/chunks.ts +58 -0
- package/src/dataset.ts +160 -0
- package/src/engine.ts +190 -0
- package/src/erddap.ts +271 -0
- package/src/extent.ts +172 -0
- package/src/fetchRetry.ts +71 -0
- package/src/index.ts +37 -0
- package/src/lobes.ts +66 -0
- package/src/mask.ts +341 -0
- package/src/preview.ts +282 -0
- package/src/sql.ts +36 -0
- package/src/sqlTemplates.ts +8 -0
- package/src/wkb.ts +81 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type { Format } from './erddap.js';
|
|
2
|
+
export interface CubeVariable {
|
|
3
|
+
name: string;
|
|
4
|
+
unit: string | null;
|
|
5
|
+
description: string;
|
|
6
|
+
categorical: boolean;
|
|
7
|
+
/** class value -> label, for categorical variables (from `erddap-places:classes`). */
|
|
8
|
+
classes?: Record<string, string>;
|
|
9
|
+
}
|
|
10
|
+
/** the long-format columns of a tabledap table: `measurement_type` / `measurement_value`. */
|
|
11
|
+
export interface LongFormat {
|
|
12
|
+
typeColumn: string;
|
|
13
|
+
valueColumn: string;
|
|
14
|
+
}
|
|
15
|
+
export interface Dataset {
|
|
16
|
+
id: string;
|
|
17
|
+
title: string;
|
|
18
|
+
baseUrl: string;
|
|
19
|
+
datasetId: string;
|
|
20
|
+
protocol: 'griddap' | 'tabledap';
|
|
21
|
+
/** set when the table keeps its variables in a column (tabledap, long format) */
|
|
22
|
+
longFormat?: LongFormat;
|
|
23
|
+
version?: string;
|
|
24
|
+
cors: boolean;
|
|
25
|
+
formats: string[];
|
|
26
|
+
lonRange: [number, number];
|
|
27
|
+
latDescending: boolean;
|
|
28
|
+
variables: CubeVariable[];
|
|
29
|
+
timeStep?: string;
|
|
30
|
+
timeExtent?: [string | null, string | null];
|
|
31
|
+
/** `erddap-places:depth`: the depth level (m) to slice on a grid with a depth axis (the surface) */
|
|
32
|
+
depth?: number;
|
|
33
|
+
/** `erddap-places:status`: 'pending' when the server does not serve it yet (list it, greyed out) */
|
|
34
|
+
status?: string;
|
|
35
|
+
format: Format;
|
|
36
|
+
collection: any;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* The SQL expression for a variable's value, converting Kelvin to °C.
|
|
40
|
+
*
|
|
41
|
+
* For a long-format table the variable is not a column: the rows were already filtered to
|
|
42
|
+
* `measurement_type = '<name>'` by the request, so the expression is the value column.
|
|
43
|
+
*/
|
|
44
|
+
export declare function valueExpr(v: CubeVariable, alias?: string, long?: LongFormat): string;
|
|
45
|
+
/** the axis label for a variable, after any unit conversion. */
|
|
46
|
+
export declare function valueLabel(v: CubeVariable): string;
|
|
47
|
+
/** the `erddap-places:categorical` flag as published in the catalog (boolean, or the string). */
|
|
48
|
+
export declare function isCategorical(v: any): boolean;
|
|
49
|
+
/** `erddap-places:long_format` -> the two column names, or undefined. */
|
|
50
|
+
export declare function toLongFormat(f: any): LongFormat | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* The SQL template a run is summarised with: point (tabledap) data rolls up by month, a flagged
|
|
53
|
+
* categorical grid gets per-class fractions, everything else is the daily grid statistics.
|
|
54
|
+
*/
|
|
55
|
+
export declare function statsTemplate(v: Pick<CubeVariable, 'categorical'> | null | undefined, protocol?: 'griddap' | 'tabledap'): 'stats_daily' | 'stats_categorical' | 'stats_tabledap';
|
|
56
|
+
/** one STAC Collection -> the fields the app uses. */
|
|
57
|
+
export declare function toDataset(c: any): Dataset;
|
|
58
|
+
/**
|
|
59
|
+
* The dataset a link names. The app writes the STAC collection id (`erddap/dhw_5km`), but links written elsewhere name
|
|
60
|
+
* ERDDAP's own dataset id (`dhw_5km`, `IMERG_monthly_global_precip`), and one collection id differs from its dataset id
|
|
61
|
+
* in case (`erddap/imerg_monthly_global_precip`). Tried in turn, each ignoring case: the collection id, the collection
|
|
62
|
+
* id with `erddap/` before it, ERDDAP's dataset id. A pending dataset is never picked; null when nothing matches.
|
|
63
|
+
*/
|
|
64
|
+
export declare function resolveDataset<D extends Pick<Dataset, 'id' | 'datasetId' | 'status'>>(datasets: D[], wanted: string): D | null;
|
|
65
|
+
/** ERDDAP wants a lon in the dataset's own range: [-180, 180] or [0, 360]. */
|
|
66
|
+
export declare function toDatasetLon(lon: number, range: [number, number]): number;
|
|
67
|
+
/**
|
|
68
|
+
* A lobe's longitude span in the dataset's own frame, [min, max], clamped to `erddap:lon_range`:
|
|
69
|
+
* a lobe that runs to 180 on a grid ending at 179.75 (CMEMS 0.25°) would otherwise ask for a
|
|
70
|
+
* longitude past the axis maximum, which ERDDAP refuses (404).
|
|
71
|
+
*/
|
|
72
|
+
export declare function lobeLonSpan(bbox: [number, number, number, number] | number[], range: [number, number]): [number, number];
|
package/dist/dataset.js
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { pickFormat } from './erddap.js';
|
|
2
|
+
const KELVIN = /^(k|kelvin|degree_?k(elvin)?)$/i;
|
|
3
|
+
/**
|
|
4
|
+
* The SQL expression for a variable's value, converting Kelvin to °C.
|
|
5
|
+
*
|
|
6
|
+
* For a long-format table the variable is not a column: the rows were already filtered to
|
|
7
|
+
* `measurement_type = '<name>'` by the request, so the expression is the value column.
|
|
8
|
+
*/
|
|
9
|
+
export function valueExpr(v, alias = 's', long) {
|
|
10
|
+
const col = long ? `${alias}."${long.valueColumn}"` : `${alias}."${v.name}"`;
|
|
11
|
+
return KELVIN.test(v.unit ?? '') ? `(${col} - 273.15)` : col;
|
|
12
|
+
}
|
|
13
|
+
/** the axis label for a variable, after any unit conversion. */
|
|
14
|
+
export function valueLabel(v) {
|
|
15
|
+
if (KELVIN.test(v.unit ?? ''))
|
|
16
|
+
return '°C';
|
|
17
|
+
const u = (v.unit ?? '').trim();
|
|
18
|
+
if (!u || u === '1' || u.toLowerCase() === 'none')
|
|
19
|
+
return v.name;
|
|
20
|
+
if (/^(celsius|degree_?c(elsius)?)$/i.test(u))
|
|
21
|
+
return '°C';
|
|
22
|
+
return u;
|
|
23
|
+
}
|
|
24
|
+
/** the `erddap-places:categorical` flag as published in the catalog (boolean, or the string). */
|
|
25
|
+
export function isCategorical(v) {
|
|
26
|
+
const f = v?.['erddap-places:categorical'];
|
|
27
|
+
return f === true || f === 'true';
|
|
28
|
+
}
|
|
29
|
+
/** `erddap-places:long_format` -> the two column names, or undefined. */
|
|
30
|
+
export function toLongFormat(f) {
|
|
31
|
+
const t = f?.type_column, v = f?.value_column;
|
|
32
|
+
return t && v ? { typeColumn: String(t), valueColumn: String(v) } : undefined;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The SQL template a run is summarised with: point (tabledap) data rolls up by month, a flagged
|
|
36
|
+
* categorical grid gets per-class fractions, everything else is the daily grid statistics.
|
|
37
|
+
*/
|
|
38
|
+
export function statsTemplate(v, protocol = 'griddap') {
|
|
39
|
+
if (protocol === 'tabledap')
|
|
40
|
+
return 'stats_tabledap';
|
|
41
|
+
return v?.categorical === true ? 'stats_categorical' : 'stats_daily';
|
|
42
|
+
}
|
|
43
|
+
function toVariable(name, v) {
|
|
44
|
+
return {
|
|
45
|
+
name,
|
|
46
|
+
unit: v?.unit ?? null,
|
|
47
|
+
description: v?.description ?? name,
|
|
48
|
+
categorical: isCategorical(v),
|
|
49
|
+
classes: v?.['erddap-places:classes'] ?? undefined,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/** one STAC Collection -> the fields the app uses. */
|
|
53
|
+
export function toDataset(c) {
|
|
54
|
+
const vars = Object.entries(c['cube:variables'] ?? {})
|
|
55
|
+
.filter(([, v]) => (v?.type ?? 'data') === 'data')
|
|
56
|
+
.map(([k, v]) => toVariable(k, v));
|
|
57
|
+
const time = c['cube:dimensions']?.time ?? {};
|
|
58
|
+
return {
|
|
59
|
+
id: c.id,
|
|
60
|
+
title: c.title ?? c.id,
|
|
61
|
+
baseUrl: c['erddap:base_url'],
|
|
62
|
+
datasetId: c['erddap:dataset_id'],
|
|
63
|
+
protocol: c['erddap:protocol'] === 'tabledap' ? 'tabledap' : 'griddap',
|
|
64
|
+
longFormat: toLongFormat(c['erddap-places:long_format']),
|
|
65
|
+
version: c['erddap:version'],
|
|
66
|
+
cors: c['erddap:cors'] !== false,
|
|
67
|
+
formats: c['erddap:formats'] ?? [],
|
|
68
|
+
lonRange: c['erddap:lon_range'] ?? [-180, 180],
|
|
69
|
+
latDescending: c['erddap:lat_descending'] === true,
|
|
70
|
+
variables: vars,
|
|
71
|
+
timeStep: time.step,
|
|
72
|
+
timeExtent: time.extent,
|
|
73
|
+
depth: Number.isFinite(Number(c['erddap-places:depth'])) && c['erddap-places:depth'] != null
|
|
74
|
+
? Number(c['erddap-places:depth']) : undefined,
|
|
75
|
+
status: c['erddap-places:status'] ?? undefined,
|
|
76
|
+
format: pickFormat({ erddap: { cors: c['erddap:cors'], formats: c['erddap:formats'] } }),
|
|
77
|
+
collection: c,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* The dataset a link names. The app writes the STAC collection id (`erddap/dhw_5km`), but links written elsewhere name
|
|
82
|
+
* ERDDAP's own dataset id (`dhw_5km`, `IMERG_monthly_global_precip`), and one collection id differs from its dataset id
|
|
83
|
+
* in case (`erddap/imerg_monthly_global_precip`). Tried in turn, each ignoring case: the collection id, the collection
|
|
84
|
+
* id with `erddap/` before it, ERDDAP's dataset id. A pending dataset is never picked; null when nothing matches.
|
|
85
|
+
*/
|
|
86
|
+
export function resolveDataset(datasets, wanted) {
|
|
87
|
+
const w = String(wanted ?? '').trim().toLowerCase();
|
|
88
|
+
if (!w)
|
|
89
|
+
return null;
|
|
90
|
+
const live = datasets.filter((d) => d.status !== 'pending');
|
|
91
|
+
const eq = (a, b) => (a ?? '').toLowerCase() === b;
|
|
92
|
+
return live.find((d) => d.id === wanted)
|
|
93
|
+
?? live.find((d) => eq(d.id, w))
|
|
94
|
+
?? live.find((d) => eq(d.id, `erddap/${w}`))
|
|
95
|
+
?? live.find((d) => eq(d.datasetId, w.replace(/^erddap\//, '')))
|
|
96
|
+
?? null;
|
|
97
|
+
}
|
|
98
|
+
/** ERDDAP wants a lon in the dataset's own range: [-180, 180] or [0, 360]. */
|
|
99
|
+
export function toDatasetLon(lon, range) {
|
|
100
|
+
if (range[1] > 180 && lon < 0)
|
|
101
|
+
return lon + 360;
|
|
102
|
+
if (range[0] < 0 && lon > 180)
|
|
103
|
+
return lon - 360;
|
|
104
|
+
return lon;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* A lobe's longitude span in the dataset's own frame, [min, max], clamped to `erddap:lon_range`:
|
|
108
|
+
* a lobe that runs to 180 on a grid ending at 179.75 (CMEMS 0.25°) would otherwise ask for a
|
|
109
|
+
* longitude past the axis maximum, which ERDDAP refuses (404).
|
|
110
|
+
*/
|
|
111
|
+
export function lobeLonSpan(bbox, range) {
|
|
112
|
+
const lo = toDatasetLon(bbox[0], range), hi = toDatasetLon(bbox[2], range);
|
|
113
|
+
const clamp = (x) => Math.min(Math.max(x, range[0]), range[1]);
|
|
114
|
+
return [clamp(Math.min(lo, hi)), clamp(Math.max(lo, hi))];
|
|
115
|
+
}
|
package/dist/engine.d.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import type * as duckdbWasm from '@duckdb/duckdb-wasm';
|
|
2
|
+
import type { MaskCell } from './mask.js';
|
|
3
|
+
import type { Params } from './sql.js';
|
|
4
|
+
/** the slice of a DuckDB-WASM database the engine uses: `AsyncDuckDB` in a browser, the node-blocking bindings in tests */
|
|
5
|
+
export interface DuckDbLike {
|
|
6
|
+
registerFileBuffer(name: string, buf: Uint8Array): unknown;
|
|
7
|
+
copyFileToBuffer(name: string): Uint8Array | Promise<Uint8Array>;
|
|
8
|
+
dropFile(name: string): unknown;
|
|
9
|
+
}
|
|
10
|
+
/** the slice of a connection the engine uses: `AsyncDuckDBConnection`, or the blocking `DuckDBConnection` */
|
|
11
|
+
export interface DuckConnLike {
|
|
12
|
+
/** an Arrow table (or a promise of one): iterable rows, `schema.fields`, `numRows` */
|
|
13
|
+
query(sql: string): any;
|
|
14
|
+
insertArrowFromIPCStream(buf: Uint8Array, opts: {
|
|
15
|
+
name: string;
|
|
16
|
+
create?: boolean;
|
|
17
|
+
}): unknown;
|
|
18
|
+
}
|
|
19
|
+
export interface Duck {
|
|
20
|
+
db: DuckDbLike;
|
|
21
|
+
conn: DuckConnLike;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The browser boot erddap-places' engine did itself: pick the bundle (eh or mvp), start its Web Worker, instantiate,
|
|
25
|
+
* connect. `duckdb` is the app's `import * as duckdb from '@duckdb/duckdb-wasm'`, `bundles` the module and worker
|
|
26
|
+
* URLs it serves (see the README, "Worker setup").
|
|
27
|
+
*/
|
|
28
|
+
export declare function connectBrowser(duckdb: typeof duckdbWasm, bundles: duckdbWasm.DuckDBBundles, logger?: duckdbWasm.Logger): Promise<Duck>;
|
|
29
|
+
export type Row = Record<string, any>;
|
|
30
|
+
/**
|
|
31
|
+
* an Arrow result as plain rows: BIGINT as a number, and DECIMAL scaled. DuckDB-WASM hands a DECIMAL back as its
|
|
32
|
+
* unscaled integer (`1.0 AS weight` in points_tabledap.sql arrived as 10, a `VALUES (20.05, ...)` latitude as 2005);
|
|
33
|
+
* erddap-places' toRows passed it through, which no ERDDAP slab (float axes, float values) ever exercised.
|
|
34
|
+
*/
|
|
35
|
+
export declare function toRows(t: any): Row[];
|
|
36
|
+
export interface EngineEvent {
|
|
37
|
+
name: string;
|
|
38
|
+
ms: number;
|
|
39
|
+
note?: string;
|
|
40
|
+
}
|
|
41
|
+
export declare class Engine {
|
|
42
|
+
db: DuckDbLike;
|
|
43
|
+
conn: DuckConnLike;
|
|
44
|
+
ready: Promise<void>;
|
|
45
|
+
marks: EngineEvent[];
|
|
46
|
+
lastSql: string;
|
|
47
|
+
private q;
|
|
48
|
+
/** `connect` runs at once (construct the engine when the app first needs it) */
|
|
49
|
+
constructor(connect: () => Promise<Duck> | Duck);
|
|
50
|
+
private mark;
|
|
51
|
+
private init;
|
|
52
|
+
/** register an already-fetched slab (parquet bytes) under a virtual file name. */
|
|
53
|
+
registerBuffer(name: string, buf: Uint8Array): Promise<any>;
|
|
54
|
+
/** forget a registered file (a slab whose statistics are done). */
|
|
55
|
+
dropFile(name: string): Promise<any>;
|
|
56
|
+
/** the gridMask cells as a DuckDB table (via Arrow IPC, so no giant VALUES list). */
|
|
57
|
+
insertMask(cells: MaskCell[], table?: string): Promise<any>;
|
|
58
|
+
/** rows from a non-Parquet slab (.csvp / .jsonp) as a table, so the same SQL works on them. */
|
|
59
|
+
insertRows(rows: Record<string, unknown>[], table: string): Promise<any>;
|
|
60
|
+
exec(sql: string, label?: string): Promise<Row[]>;
|
|
61
|
+
/**
|
|
62
|
+
* The result table as Parquet bytes, written by DuckDB itself.
|
|
63
|
+
*
|
|
64
|
+
* The rows go back into a temporary table (Arrow IPC, as the mask does), `COPY … TO` writes a
|
|
65
|
+
* Parquet file into the WASM filesystem and `copyFileToBuffer` hands it back. Going through the
|
|
66
|
+
* rows rather than re-running the statistics keeps the download identical to what is on screen,
|
|
67
|
+
* whatever has happened to the registered slab since.
|
|
68
|
+
*/
|
|
69
|
+
toParquet(rows: Record<string, unknown>[], table?: string, file?: string): Promise<Uint8Array>;
|
|
70
|
+
/** render a `sql/<name>.sql` template with {{params}} and run it. */
|
|
71
|
+
runTemplate(name: string, params: Params): Promise<Row[]>;
|
|
72
|
+
}
|
package/dist/engine.js
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
// DuckDB-WASM as the statistics engine: the slab goes in as a registered Parquet buffer, the mask as an Arrow table,
|
|
2
|
+
// and every statistic is a sql/ template rendered with {{named}} params. moved from erddap-places
|
|
3
|
+
// `app/src/lib/engine.ts`; the methods are unchanged, but the package never boots DuckDB itself: the app owns the
|
|
4
|
+
// bundles and the Web Worker (Vite `?url` imports, a CDN, or the node-blocking bundle in tests) and hands the Engine a
|
|
5
|
+
// `connect()` that resolves to `{ db, conn }`. `connectBrowser(duckdb, bundles)` is that boot for a browser app, with the
|
|
6
|
+
// duckdb-wasm module passed in (a peer dependency, never imported at run time here).
|
|
7
|
+
import { DataType, tableFromArrays, tableToIPC } from 'apache-arrow';
|
|
8
|
+
import { render } from './sql.js';
|
|
9
|
+
/**
|
|
10
|
+
* The browser boot erddap-places' engine did itself: pick the bundle (eh or mvp), start its Web Worker, instantiate,
|
|
11
|
+
* connect. `duckdb` is the app's `import * as duckdb from '@duckdb/duckdb-wasm'`, `bundles` the module and worker
|
|
12
|
+
* URLs it serves (see the README, "Worker setup").
|
|
13
|
+
*/
|
|
14
|
+
export async function connectBrowser(duckdb, bundles, logger = new duckdb.VoidLogger()) {
|
|
15
|
+
const bundle = await duckdb.selectBundle(bundles);
|
|
16
|
+
const worker = new Worker(bundle.mainWorker);
|
|
17
|
+
const db = new duckdb.AsyncDuckDB(logger, worker);
|
|
18
|
+
await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
|
|
19
|
+
return { db, conn: await db.connect() };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* an Arrow result as plain rows: BIGINT as a number, and DECIMAL scaled. DuckDB-WASM hands a DECIMAL back as its
|
|
23
|
+
* unscaled integer (`1.0 AS weight` in points_tabledap.sql arrived as 10, a `VALUES (20.05, ...)` latitude as 2005);
|
|
24
|
+
* erddap-places' toRows passed it through, which no ERDDAP slab (float axes, float values) ever exercised.
|
|
25
|
+
*/
|
|
26
|
+
export function toRows(t) {
|
|
27
|
+
const cols = t.schema.fields.map((f) => f.name);
|
|
28
|
+
const scale = t.schema.fields.map((f) => (DataType.isDecimal(f.type) ? 10 ** f.type.scale : 0));
|
|
29
|
+
const out = [];
|
|
30
|
+
for (const r of t) {
|
|
31
|
+
const o = {};
|
|
32
|
+
cols.forEach((c, i) => {
|
|
33
|
+
const v = r[c];
|
|
34
|
+
o[c] = typeof v === 'bigint' ? Number(v) : scale[i] && v !== null && v !== undefined ? Number(v) / scale[i] : v;
|
|
35
|
+
});
|
|
36
|
+
out.push(o);
|
|
37
|
+
}
|
|
38
|
+
return out;
|
|
39
|
+
}
|
|
40
|
+
const now = () => (globalThis.performance ?? Date).now();
|
|
41
|
+
export class Engine {
|
|
42
|
+
db;
|
|
43
|
+
conn;
|
|
44
|
+
ready;
|
|
45
|
+
marks = [];
|
|
46
|
+
lastSql = '';
|
|
47
|
+
q = Promise.resolve(); // one connection: serialize everything
|
|
48
|
+
/** `connect` runs at once (construct the engine when the app first needs it) */
|
|
49
|
+
constructor(connect) { this.ready = this.init(connect); }
|
|
50
|
+
mark(name, ms, note) { this.marks.push({ name, ms: Math.round(ms), note }); }
|
|
51
|
+
async init(connect) {
|
|
52
|
+
const t = now();
|
|
53
|
+
const { db, conn } = await connect();
|
|
54
|
+
this.db = db;
|
|
55
|
+
this.conn = conn;
|
|
56
|
+
this.mark('connect', now() - t);
|
|
57
|
+
}
|
|
58
|
+
/** register an already-fetched slab (parquet bytes) under a virtual file name. */
|
|
59
|
+
async registerBuffer(name, buf) {
|
|
60
|
+
const run = async () => {
|
|
61
|
+
await this.ready;
|
|
62
|
+
const t = now();
|
|
63
|
+
await this.db.registerFileBuffer(name, buf);
|
|
64
|
+
this.mark(`register:${name}`, now() - t, `${(buf.byteLength / 1e3).toFixed(0)} kB`);
|
|
65
|
+
};
|
|
66
|
+
this.q = this.q.then(run, run);
|
|
67
|
+
return this.q;
|
|
68
|
+
}
|
|
69
|
+
/** forget a registered file (a slab whose statistics are done). */
|
|
70
|
+
async dropFile(name) {
|
|
71
|
+
const run = async () => {
|
|
72
|
+
await this.ready;
|
|
73
|
+
await this.db.dropFile(name);
|
|
74
|
+
};
|
|
75
|
+
this.q = this.q.then(run, run);
|
|
76
|
+
return this.q;
|
|
77
|
+
}
|
|
78
|
+
/** the gridMask cells as a DuckDB table (via Arrow IPC, so no giant VALUES list). */
|
|
79
|
+
async insertMask(cells, table = 'mask') {
|
|
80
|
+
const run = async () => {
|
|
81
|
+
await this.ready;
|
|
82
|
+
const t = now();
|
|
83
|
+
const arrow = tableFromArrays({
|
|
84
|
+
latitude: Float64Array.from(cells.map((c) => c.lat)),
|
|
85
|
+
longitude: Float64Array.from(cells.map((c) => c.lon)),
|
|
86
|
+
weight: Float64Array.from(cells.map((c) => c.weight)),
|
|
87
|
+
});
|
|
88
|
+
await this.conn.query(`DROP TABLE IF EXISTS ${table}`);
|
|
89
|
+
await this.conn.insertArrowFromIPCStream(tableToIPC(arrow, 'stream'), { name: table, create: true });
|
|
90
|
+
this.mark(`mask:${table}`, now() - t, `${cells.length} cells`);
|
|
91
|
+
};
|
|
92
|
+
this.q = this.q.then(run, run);
|
|
93
|
+
return this.q;
|
|
94
|
+
}
|
|
95
|
+
/** rows from a non-Parquet slab (.csvp / .jsonp) as a table, so the same SQL works on them. */
|
|
96
|
+
async insertRows(rows, table) {
|
|
97
|
+
const run = async () => {
|
|
98
|
+
await this.ready;
|
|
99
|
+
const t = now();
|
|
100
|
+
const cols = Object.keys(rows[0] ?? {});
|
|
101
|
+
const arrays = {};
|
|
102
|
+
for (const c of cols) {
|
|
103
|
+
const vals = rows.map((r) => r[c]);
|
|
104
|
+
arrays[c] = vals.every((v) => typeof v === 'number' || v === null)
|
|
105
|
+
? Float64Array.from(vals.map((v) => (v === null ? NaN : v)))
|
|
106
|
+
: vals.map((v) => (v === null || v === undefined ? null : String(v)));
|
|
107
|
+
}
|
|
108
|
+
await this.conn.query(`DROP TABLE IF EXISTS ${table}`);
|
|
109
|
+
await this.conn.insertArrowFromIPCStream(tableToIPC(tableFromArrays(arrays), 'stream'), { name: table, create: true });
|
|
110
|
+
this.mark(`rows:${table}`, now() - t, `${rows.length} rows`);
|
|
111
|
+
};
|
|
112
|
+
this.q = this.q.then(run, run);
|
|
113
|
+
return this.q;
|
|
114
|
+
}
|
|
115
|
+
exec(sql, label = 'exec') {
|
|
116
|
+
const run = async () => {
|
|
117
|
+
await this.ready;
|
|
118
|
+
const t = now();
|
|
119
|
+
this.lastSql = sql;
|
|
120
|
+
const res = await this.conn.query(sql);
|
|
121
|
+
this.mark(`query:${label}`, now() - t, `${res.numRows} rows`);
|
|
122
|
+
return toRows(res);
|
|
123
|
+
};
|
|
124
|
+
this.q = this.q.then(run, run);
|
|
125
|
+
return this.q;
|
|
126
|
+
}
|
|
127
|
+
/**
|
|
128
|
+
* The result table as Parquet bytes, written by DuckDB itself.
|
|
129
|
+
*
|
|
130
|
+
* The rows go back into a temporary table (Arrow IPC, as the mask does), `COPY … TO` writes a
|
|
131
|
+
* Parquet file into the WASM filesystem and `copyFileToBuffer` hands it back. Going through the
|
|
132
|
+
* rows rather than re-running the statistics keeps the download identical to what is on screen,
|
|
133
|
+
* whatever has happened to the registered slab since.
|
|
134
|
+
*/
|
|
135
|
+
async toParquet(rows, table = '_export', file = 'export.parquet') {
|
|
136
|
+
if (!rows.length)
|
|
137
|
+
throw new Error('nothing to export');
|
|
138
|
+
await this.insertRows(rows, table);
|
|
139
|
+
const run = async () => {
|
|
140
|
+
await this.ready;
|
|
141
|
+
const t = now();
|
|
142
|
+
await this.conn.query(`COPY (SELECT * FROM ${table}) TO '${file}' (FORMAT PARQUET)`);
|
|
143
|
+
const buf = await this.db.copyFileToBuffer(file);
|
|
144
|
+
await Promise.resolve(this.db.dropFile(file)).catch(() => { });
|
|
145
|
+
await this.conn.query(`DROP TABLE IF EXISTS ${table}`);
|
|
146
|
+
this.mark(`parquet:${file}`, now() - t, `${rows.length} rows, ${(buf.byteLength / 1e3).toFixed(0)} kB`);
|
|
147
|
+
return buf;
|
|
148
|
+
};
|
|
149
|
+
this.q = this.q.then(run, run);
|
|
150
|
+
return this.q;
|
|
151
|
+
}
|
|
152
|
+
/** render a `sql/<name>.sql` template with {{params}} and run it. */
|
|
153
|
+
runTemplate(name, params) { return this.exec(render(name, params), name); }
|
|
154
|
+
}
|
package/dist/erddap.d.ts
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { type RetryOpts } from './fetchRetry.js';
|
|
2
|
+
export type Format = 'parquet' | 'parquetWMeta' | 'csvp' | 'jsonp';
|
|
3
|
+
/** the two Parquet rungs are handled identically once the bytes are in hand. */
|
|
4
|
+
export declare const isParquet: (f: Format) => f is "parquet" | "parquetWMeta";
|
|
5
|
+
export interface ErddapMeta {
|
|
6
|
+
version?: string;
|
|
7
|
+
cors?: boolean;
|
|
8
|
+
formats?: string[];
|
|
9
|
+
}
|
|
10
|
+
export interface Collection {
|
|
11
|
+
erddap: ErddapMeta;
|
|
12
|
+
}
|
|
13
|
+
export interface GriddapSpec {
|
|
14
|
+
base: string;
|
|
15
|
+
datasetId: string;
|
|
16
|
+
variable: string;
|
|
17
|
+
time: [string, string];
|
|
18
|
+
lat: [number, number];
|
|
19
|
+
lon: [number, number];
|
|
20
|
+
latDescending?: boolean;
|
|
21
|
+
/** a depth axis between time and latitude (CMEMS): the single level to slice, e.g. 0.494 (surface) */
|
|
22
|
+
depth?: number;
|
|
23
|
+
format?: Format;
|
|
24
|
+
callback?: string;
|
|
25
|
+
}
|
|
26
|
+
/** one axis constraint, `[(lo):1:(hi)]` with the brackets encoded. */
|
|
27
|
+
export declare function constraint(lo: string | number, hi: string | number, stride?: number): string;
|
|
28
|
+
/** an ISO instant at 12:00Z for a Date or a yyyy-mm-dd string (ERDDAP daily grids sit at noon). */
|
|
29
|
+
export declare function noonZ(d: Date | string): string;
|
|
30
|
+
/** the griddap request URL for one lobe. */
|
|
31
|
+
export declare function griddapUrl(s: GriddapSpec): string;
|
|
32
|
+
/**
|
|
33
|
+
* the griddap URL for ONE time step (`[(t):1:(t)]`), for the map when the statistics come from the
|
|
34
|
+
* precomputed file: the same axis order, longitude range and depth as griddapUrl(), but a single step
|
|
35
|
+
* (`at` is the ISO instant of that step, e.g. the dataset's last time).
|
|
36
|
+
*/
|
|
37
|
+
export declare function griddapSliceUrl(s: Omit<GriddapSpec, 'time'> & {
|
|
38
|
+
at: string;
|
|
39
|
+
}): string;
|
|
40
|
+
/** one URL per polygon lobe (bbox = [lonMin, latMin, lonMax, latMax]). */
|
|
41
|
+
export declare function griddapUrls(s: Omit<GriddapSpec, 'lat' | 'lon'>, bboxes: Array<[number, number, number, number]>): string[];
|
|
42
|
+
export type TableOp = '>=' | '<=' | '>' | '<' | '=' | '!=' | '=~';
|
|
43
|
+
export interface TableConstraint {
|
|
44
|
+
column: string;
|
|
45
|
+
op: TableOp;
|
|
46
|
+
value: string | number;
|
|
47
|
+
}
|
|
48
|
+
export interface TabledapSpec {
|
|
49
|
+
base: string;
|
|
50
|
+
datasetId: string;
|
|
51
|
+
columns: string[];
|
|
52
|
+
constraints?: TableConstraint[];
|
|
53
|
+
format?: Format;
|
|
54
|
+
callback?: string;
|
|
55
|
+
/** extra server-side functions, e.g. `distinct()` or `orderByMax("time")`, appended literally. */
|
|
56
|
+
functions?: string[];
|
|
57
|
+
}
|
|
58
|
+
/** one tabledap constraint, `column%3E=value`. */
|
|
59
|
+
export declare function tableConstraint(c: TableConstraint): string;
|
|
60
|
+
/** the tabledap request URL: the encoded column list, then one `&constraint` each. */
|
|
61
|
+
export declare function tabledapUrl(s: TabledapSpec): string;
|
|
62
|
+
/**
|
|
63
|
+
* The constraints for one place lobe and one time window: a bbox and a half-open-ish time range,
|
|
64
|
+
* plus the long-format row filter when the dataset keeps its variables in a `measurement_type`
|
|
65
|
+
* column (`erddap-places:long_format` in the STAC Collection).
|
|
66
|
+
*/
|
|
67
|
+
export declare function tabledapPlaceConstraints(o: {
|
|
68
|
+
bbox: [number, number, number, number];
|
|
69
|
+
from: string;
|
|
70
|
+
to: string;
|
|
71
|
+
typeColumn?: string;
|
|
72
|
+
typeValue?: string;
|
|
73
|
+
}): TableConstraint[];
|
|
74
|
+
/**
|
|
75
|
+
* per-attempt timeout of an axis request. small, but a busy server is slow whatever the size: PacIOOS answered
|
|
76
|
+
* `dhw_5km` latitude requests in 13–19 s on 2026-10-20, past the old 15 s, so a live run failed while the server
|
|
77
|
+
* was up ("did not answer (no response within 15 s, 2 tries)").
|
|
78
|
+
*/
|
|
79
|
+
export declare const AXIS_TIMEOUT_MS = 45000;
|
|
80
|
+
/** per-attempt timeout of the map's one-step slice: as small, and as slow on a busy server (PacIOOS took 27–31 s for 45 days of one pixel) */
|
|
81
|
+
export declare const SLICE_TIMEOUT_MS = 45000;
|
|
82
|
+
/**
|
|
83
|
+
* `dataset.json?latitude[(min):1:(max)]` -> the axis values, in the server's own order. `AXIS_TIMEOUT_MS` per
|
|
84
|
+
* attempt and one retry by default (`retry` overrides them; the precompute waits longer).
|
|
85
|
+
*/
|
|
86
|
+
export declare function fetchAxis(base: string, datasetId: string, axis: string, min: number, max: number, descending?: boolean, signal?: AbortSignal, retry?: Omit<RetryOpts, 'signal'>): Promise<number[]>;
|
|
87
|
+
/**
|
|
88
|
+
* Best format this server can serve us: Parquet -> CSV (`.csvp`) -> JSONP.
|
|
89
|
+
* CORS off means only JSONP (a `<script>` tag is not subject to CORS).
|
|
90
|
+
*/
|
|
91
|
+
export declare function pickFormat(collection: Collection): Format;
|
|
92
|
+
export interface Slab {
|
|
93
|
+
url: string;
|
|
94
|
+
format: Format;
|
|
95
|
+
ms: number;
|
|
96
|
+
bytes?: number;
|
|
97
|
+
buffer?: Uint8Array;
|
|
98
|
+
rows?: Record<string, unknown>[];
|
|
99
|
+
text?: string;
|
|
100
|
+
}
|
|
101
|
+
/** load a `.json?...&.jsonp=cb` URL through a <script> tag (browser only; no CORS needed). */
|
|
102
|
+
export declare function fetchJsonp(url: string, cbName: string, timeoutMs?: number, signal?: AbortSignal): Promise<any>;
|
|
103
|
+
/** `{table:{columnNames, rows}}` -> array of objects. */
|
|
104
|
+
export declare function tableToRows(t: {
|
|
105
|
+
table: {
|
|
106
|
+
columnNames: string[];
|
|
107
|
+
rows: unknown[][];
|
|
108
|
+
};
|
|
109
|
+
}): Record<string, unknown>[];
|
|
110
|
+
/** minimal CSV parse for `.csvp` (ERDDAP quotes with " and doubles inner quotes). */
|
|
111
|
+
export declare function parseCsvp(text: string): Record<string, unknown>[];
|
|
112
|
+
/**
|
|
113
|
+
* fetch one slab in the given format. Parquet comes back as bytes for `registerFileBuffer`. `retry`
|
|
114
|
+
* sets the per-attempt timeout and the tries (fetchRetry: 20 s and one retry unless the caller says
|
|
115
|
+
* otherwise: a whole live window can take longer than the one-step map slice).
|
|
116
|
+
*/
|
|
117
|
+
export declare function fetchSlab(url: string, format?: Format, callback?: string, signal?: AbortSignal, retry?: Omit<RetryOpts, 'signal'>): Promise<Slab>;
|