@oceanmetrics/engine 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/LICENSE +21 -0
- package/README.md +192 -2
- package/dist/cells.d.ts +84 -0
- package/dist/cells.js +179 -0
- package/dist/chunks.d.ts +29 -0
- package/dist/chunks.js +49 -0
- package/dist/dataset.d.ts +72 -0
- package/dist/dataset.js +115 -0
- package/dist/engine.d.ts +72 -0
- package/dist/engine.js +154 -0
- package/dist/erddap.d.ts +117 -0
- package/dist/erddap.js +224 -0
- package/dist/extent.d.ts +64 -0
- package/dist/extent.js +151 -0
- package/dist/fetchRetry.d.ts +33 -0
- package/dist/fetchRetry.js +76 -0
- package/dist/index.d.ts +24 -0
- package/dist/index.js +16 -0
- package/dist/lobes.d.ts +24 -0
- package/dist/lobes.js +55 -0
- package/dist/mask.d.ts +71 -0
- package/dist/mask.js +363 -0
- package/dist/preview.d.ts +121 -0
- package/dist/preview.js +202 -0
- package/dist/sql.d.ts +10 -0
- package/dist/sql.js +33 -0
- package/dist/sqlTemplates.d.ts +1 -0
- package/dist/sqlTemplates.js +8 -0
- package/dist/wkb.d.ts +5 -0
- package/dist/wkb.js +80 -0
- package/package.json +71 -4
- package/sql/last_step.sql +31 -0
- package/sql/points_tabledap.sql +21 -0
- package/sql/stats_categorical.sql +36 -0
- package/sql/stats_daily.sql +27 -0
- package/sql/stats_tabledap.sql +36 -0
- package/src/cells.ts +190 -0
- package/src/chunks.ts +58 -0
- package/src/dataset.ts +160 -0
- package/src/engine.ts +190 -0
- package/src/erddap.ts +271 -0
- package/src/extent.ts +172 -0
- package/src/fetchRetry.ts +71 -0
- package/src/index.ts +37 -0
- package/src/lobes.ts +66 -0
- package/src/mask.ts +341 -0
- package/src/preview.ts +279 -0
- package/src/sql.ts +36 -0
- package/src/sqlTemplates.ts +8 -0
- package/src/wkb.ts +81 -0
package/src/erddap.ts
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
// ERDDAP griddap client. no proxy of ours in the path: the browser talks to the ERDDAP server.
|
|
2
|
+
//
|
|
3
|
+
// URL shape (griddap): {base}/griddap/{dataset}.{fmt}?{var}[(t0):1:(t1)][(lat0):1:(lat1)][(lon0):1:(lon1)]
|
|
4
|
+
// each constraint's square brackets are percent-encoded (%5B / %5D); the rest of the query is left
|
|
5
|
+
// literal, because ERDDAP will not accept a fully encoded query string. constraint values go in the
|
|
6
|
+
// axis's own direction (latitude descends on the CRW grid, so [(22.4):1:(18.8)]). a 4-D grid
|
|
7
|
+
// (time, depth, latitude, longitude) gets one more constraint, a single depth level, after time.
|
|
8
|
+
|
|
9
|
+
import { fetchRetry, type RetryOpts } from './fetchRetry.js'
|
|
10
|
+
|
|
11
|
+
export type Format = 'parquet' | 'parquetWMeta' | 'csvp' | 'jsonp'
|
|
12
|
+
/** the two Parquet rungs are handled identically once the bytes are in hand. */
|
|
13
|
+
export const isParquet = (f: Format) => f === 'parquet' || f === 'parquetWMeta'
|
|
14
|
+
|
|
15
|
+
export interface ErddapMeta { version?: string; cors?: boolean; formats?: string[] }
|
|
16
|
+
export interface Collection { erddap: ErddapMeta }
|
|
17
|
+
|
|
18
|
+
export interface GriddapSpec {
|
|
19
|
+
base : string
|
|
20
|
+
datasetId : string
|
|
21
|
+
variable : string
|
|
22
|
+
time : [string, string]
|
|
23
|
+
lat : [number, number] // [min, max], any order; reordered per latDescending
|
|
24
|
+
lon : [number, number]
|
|
25
|
+
latDescending?: boolean
|
|
26
|
+
/** a depth axis between time and latitude (CMEMS): the single level to slice, e.g. 0.494 (surface) */
|
|
27
|
+
depth ?: number
|
|
28
|
+
format ?: Format
|
|
29
|
+
callback ?: string // jsonp only
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const enc = (s: string) => s.replace(/\[/g, '%5B').replace(/\]/g, '%5D')
|
|
33
|
+
const stripSlash = (s: string) => s.replace(/\/+$/, '')
|
|
34
|
+
/** one axis constraint, `[(lo):1:(hi)]` with the brackets encoded. */
|
|
35
|
+
export function constraint(lo: string | number, hi: string | number, stride = 1): string {
|
|
36
|
+
return enc(`[(${lo}):${stride}:(${hi})]`)
|
|
37
|
+
}
|
|
38
|
+
/** an ISO instant at 12:00Z for a Date or a yyyy-mm-dd string (ERDDAP daily grids sit at noon). */
|
|
39
|
+
export function noonZ(d: Date | string): string {
|
|
40
|
+
const iso = typeof d === 'string' ? d.slice(0, 10) : d.toISOString().slice(0, 10)
|
|
41
|
+
return `${iso}T12:00:00Z`
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const EXT: Record<Format, string> = {
|
|
45
|
+
parquet: '.parquet', parquetWMeta: '.parquetWMeta', csvp: '.csvp', jsonp: '.json',
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** the griddap request URL for one lobe. */
|
|
49
|
+
export function griddapUrl(s: GriddapSpec): string {
|
|
50
|
+
const fmt = s.format ?? 'parquet'
|
|
51
|
+
const desc = s.latDescending ?? true
|
|
52
|
+
const [y0, y1] = [Math.min(...s.lat), Math.max(...s.lat)]
|
|
53
|
+
const [x0, x1] = [Math.min(...s.lon), Math.max(...s.lon)]
|
|
54
|
+
const q =
|
|
55
|
+
s.variable +
|
|
56
|
+
constraint(s.time[0], s.time[1]) +
|
|
57
|
+
(s.depth != null && Number.isFinite(s.depth) ? constraint(s.depth, s.depth) : '') +
|
|
58
|
+
(desc ? constraint(y1, y0) : constraint(y0, y1)) +
|
|
59
|
+
constraint(x0, x1)
|
|
60
|
+
const tail = fmt === 'jsonp' ? `&.jsonp=${s.callback ?? 'erddapCb'}` : ''
|
|
61
|
+
return `${stripSlash(s.base)}/griddap/${s.datasetId}${EXT[fmt]}?${q}${tail}`
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* the griddap URL for ONE time step (`[(t):1:(t)]`), for the map when the statistics come from the
|
|
66
|
+
* precomputed file: the same axis order, longitude range and depth as griddapUrl(), but a single step
|
|
67
|
+
* (`at` is the ISO instant of that step, e.g. the dataset's last time).
|
|
68
|
+
*/
|
|
69
|
+
export function griddapSliceUrl(s: Omit<GriddapSpec, 'time'> & { at: string }): string {
|
|
70
|
+
const { at, ...rest } = s
|
|
71
|
+
return griddapUrl({ ...rest, time: [at, at] })
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** one URL per polygon lobe (bbox = [lonMin, latMin, lonMax, latMax]). */
|
|
75
|
+
export function griddapUrls(s: Omit<GriddapSpec, 'lat' | 'lon'>, bboxes: Array<[number, number, number, number]>): string[] {
|
|
76
|
+
return bboxes.map((b) => griddapUrl({ ...s, lon: [b[0], b[2]], lat: [b[1], b[3]] }))
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// ── tabledap ──────────────────────────────────────────────────────────────────
|
|
80
|
+
// URL shape: {base}/tabledap/{dataset}.{fmt}?{col},{col},…&{col}{op}{value}&…
|
|
81
|
+
// the column list is comma-separated and ERDDAP wants the commas percent-encoded (%2C); each
|
|
82
|
+
// constraint's operator is encoded too (%3E= / %3C= / %3D), and the values are left literal, which
|
|
83
|
+
// is what an ERDDAP 2.30 server accepts (verified against erddap.calcofi.io).
|
|
84
|
+
|
|
85
|
+
export type TableOp = '>=' | '<=' | '>' | '<' | '=' | '!=' | '=~'
|
|
86
|
+
export interface TableConstraint { column: string; op: TableOp; value: string | number }
|
|
87
|
+
export interface TabledapSpec {
|
|
88
|
+
base : string
|
|
89
|
+
datasetId : string
|
|
90
|
+
columns : string[]
|
|
91
|
+
constraints?: TableConstraint[]
|
|
92
|
+
format ?: Format
|
|
93
|
+
callback ?: string
|
|
94
|
+
/** extra server-side functions, e.g. `distinct()` or `orderByMax("time")`, appended literally. */
|
|
95
|
+
functions ?: string[]
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// only the comparison characters are encoded; `=` stays literal, as ERDDAP's own example URLs do
|
|
99
|
+
const OPS: Record<TableOp, string> = {
|
|
100
|
+
'>=': '%3E=', '<=': '%3C=', '>': '%3E', '<': '%3C', '=': '=', '!=': '!=', '=~': '=~',
|
|
101
|
+
}
|
|
102
|
+
/** one tabledap constraint, `column%3E=value`. */
|
|
103
|
+
export function tableConstraint(c: TableConstraint): string {
|
|
104
|
+
return `${c.column}${OPS[c.op] ?? c.op}${c.value}`
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** the tabledap request URL: the encoded column list, then one `&constraint` each. */
|
|
108
|
+
export function tabledapUrl(s: TabledapSpec): string {
|
|
109
|
+
const fmt = s.format ?? 'parquetWMeta'
|
|
110
|
+
const q = [
|
|
111
|
+
s.columns.join('%2C'),
|
|
112
|
+
...(s.constraints ?? []).map(tableConstraint),
|
|
113
|
+
...(s.functions ?? []),
|
|
114
|
+
]
|
|
115
|
+
const tail = fmt === 'jsonp' ? `&.jsonp=${s.callback ?? 'erddapCb'}` : ''
|
|
116
|
+
return `${stripSlash(s.base)}/tabledap/${s.datasetId}${EXT[fmt]}?${q.join('&')}${tail}`
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* The constraints for one place lobe and one time window: a bbox and a half-open-ish time range,
|
|
121
|
+
* plus the long-format row filter when the dataset keeps its variables in a `measurement_type`
|
|
122
|
+
* column (`erddap-places:long_format` in the STAC Collection).
|
|
123
|
+
*/
|
|
124
|
+
export function tabledapPlaceConstraints(o: {
|
|
125
|
+
bbox: [number, number, number, number] // [lonMin, latMin, lonMax, latMax]
|
|
126
|
+
from: string // yyyy-mm-dd or an ISO instant
|
|
127
|
+
to : string
|
|
128
|
+
typeColumn?: string
|
|
129
|
+
typeValue ?: string
|
|
130
|
+
}): TableConstraint[] {
|
|
131
|
+
const iso = (d: string, end = false) => (d.length > 10 ? d : `${d}T${end ? '23:59:59' : '00:00:00'}Z`)
|
|
132
|
+
const cs: TableConstraint[] = [
|
|
133
|
+
{ column: 'time', op: '>=', value: iso(o.from) },
|
|
134
|
+
{ column: 'time', op: '<=', value: iso(o.to, true) },
|
|
135
|
+
{ column: 'longitude', op: '>=', value: o.bbox[0] },
|
|
136
|
+
{ column: 'longitude', op: '<=', value: o.bbox[2] },
|
|
137
|
+
{ column: 'latitude', op: '>=', value: o.bbox[1] },
|
|
138
|
+
{ column: 'latitude', op: '<=', value: o.bbox[3] },
|
|
139
|
+
]
|
|
140
|
+
// a string value is quoted, and the quotes percent-encoded (%22), as ERDDAP requires
|
|
141
|
+
if (o.typeColumn && o.typeValue) cs.push({ column: o.typeColumn, op: '=', value: `%22${o.typeValue}%22` })
|
|
142
|
+
return cs
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// ── axis vectors ──────────────────────────────────────────────────────────────
|
|
146
|
+
/**
|
|
147
|
+
* per-attempt timeout of an axis request. small, but a busy server is slow whatever the size: PacIOOS answered
|
|
148
|
+
* `dhw_5km` latitude requests in 13–19 s on 2026-10-20, past the old 15 s, so a live run failed while the server
|
|
149
|
+
* was up ("did not answer (no response within 15 s, 2 tries)").
|
|
150
|
+
*/
|
|
151
|
+
export const AXIS_TIMEOUT_MS = 45_000
|
|
152
|
+
/** per-attempt timeout of the map's one-step slice: as small, and as slow on a busy server (PacIOOS took 27–31 s for 45 days of one pixel) */
|
|
153
|
+
export const SLICE_TIMEOUT_MS = 45_000
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* `dataset.json?latitude[(min):1:(max)]` -> the axis values, in the server's own order. `AXIS_TIMEOUT_MS` per
|
|
157
|
+
* attempt and one retry by default (`retry` overrides them; the precompute waits longer).
|
|
158
|
+
*/
|
|
159
|
+
export async function fetchAxis(base: string, datasetId: string, axis: string, min: number, max: number, descending = false,
|
|
160
|
+
signal?: AbortSignal, retry: Omit<RetryOpts, 'signal'> = {}): Promise<number[]> {
|
|
161
|
+
const c = descending ? constraint(max, min) : constraint(min, max)
|
|
162
|
+
const url = `${stripSlash(base)}/griddap/${datasetId}.json?${axis}${c}`
|
|
163
|
+
return fetchRetry(url, { timeoutMs: AXIS_TIMEOUT_MS, ...retry, signal }, async (res) => {
|
|
164
|
+
if (!res.ok) throw new Error(`${url}: ${res.status} ${res.statusText}`)
|
|
165
|
+
const j: { table: { columnNames: string[]; rows: number[][] } } = await res.json()
|
|
166
|
+
return j.table.rows.map((r) => Number(r[0]))
|
|
167
|
+
})
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
// ── format rung ───────────────────────────────────────────────────────────────
|
|
171
|
+
/**
|
|
172
|
+
* Best format this server can serve us: Parquet -> CSV (`.csvp`) -> JSONP.
|
|
173
|
+
* CORS off means only JSONP (a `<script>` tag is not subject to CORS).
|
|
174
|
+
*/
|
|
175
|
+
export function pickFormat(collection: Collection): Format {
|
|
176
|
+
const e = collection?.erddap ?? {}
|
|
177
|
+
const fmts = (e.formats ?? []).map((f) => f.replace(/^\./, '').toLowerCase())
|
|
178
|
+
if (e.cors === false) return 'jsonp'
|
|
179
|
+
if (fmts.length === 0) return 'parquet'
|
|
180
|
+
// parquetWMeta is plain Parquet with ERDDAP's metadata in the file footer: prefer it when offered
|
|
181
|
+
if (fmts.includes('parquetwmeta')) return 'parquetWMeta'
|
|
182
|
+
if (fmts.includes('parquet')) return 'parquet'
|
|
183
|
+
if (fmts.includes('csvp') || fmts.includes('csv')) return 'csvp'
|
|
184
|
+
return 'jsonp'
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ── fetch ─────────────────────────────────────────────────────────────────────
|
|
188
|
+
export interface Slab {
|
|
189
|
+
url : string
|
|
190
|
+
format : Format
|
|
191
|
+
ms : number
|
|
192
|
+
bytes ?: number
|
|
193
|
+
buffer?: Uint8Array // parquet
|
|
194
|
+
rows ?: Record<string, unknown>[] // csvp / jsonp
|
|
195
|
+
text ?: string // csvp, raw
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** load a `.json?...&.jsonp=cb` URL through a <script> tag (browser only; no CORS needed). */
|
|
199
|
+
export function fetchJsonp(url: string, cbName: string, timeoutMs = 120_000, signal?: AbortSignal): Promise<any> {
|
|
200
|
+
return new Promise((resolve, reject) => {
|
|
201
|
+
if (typeof document === 'undefined') return reject(new Error('jsonp needs a browser document'))
|
|
202
|
+
const done = (fn: () => void) => { clearTimeout(timer); delete (window as any)[cbName]; script.remove(); fn() }
|
|
203
|
+
const timer = setTimeout(() => done(() => reject(new Error(`jsonp timeout: ${url}`))), timeoutMs)
|
|
204
|
+
;(window as any)[cbName] = (data: any) => done(() => resolve(data))
|
|
205
|
+
// a <script> tag cannot be cancelled, but a superseded run must stop waiting on it
|
|
206
|
+
signal?.addEventListener('abort', () => done(() => reject(new DOMException('aborted', 'AbortError'))), { once: true })
|
|
207
|
+
const script = document.createElement('script')
|
|
208
|
+
script.src = url
|
|
209
|
+
script.onerror = () => done(() => reject(new Error(`jsonp failed: ${url}`)))
|
|
210
|
+
document.head.appendChild(script)
|
|
211
|
+
})
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** `{table:{columnNames, rows}}` -> array of objects. */
|
|
215
|
+
export function tableToRows(t: { table: { columnNames: string[]; rows: unknown[][] } }): Record<string, unknown>[] {
|
|
216
|
+
const { columnNames, rows } = t.table
|
|
217
|
+
return rows.map((r) => Object.fromEntries(columnNames.map((c, i) => [c, r[i]])))
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** minimal CSV parse for `.csvp` (ERDDAP quotes with " and doubles inner quotes). */
|
|
221
|
+
export function parseCsvp(text: string): Record<string, unknown>[] {
|
|
222
|
+
const lines = text.split('\n').filter((l) => l.length)
|
|
223
|
+
if (!lines.length) return []
|
|
224
|
+
const split = (l: string) => {
|
|
225
|
+
const out: string[] = []; let cur = '', q = false
|
|
226
|
+
for (let i = 0; i < l.length; i++) {
|
|
227
|
+
const ch = l[i]
|
|
228
|
+
if (q) { if (ch === '"') { if (l[i + 1] === '"') { cur += '"'; i++ } else q = false } else cur += ch }
|
|
229
|
+
else if (ch === '"') q = true
|
|
230
|
+
else if (ch === ',') { out.push(cur); cur = '' }
|
|
231
|
+
else if (ch !== '\r') cur += ch
|
|
232
|
+
}
|
|
233
|
+
out.push(cur); return out
|
|
234
|
+
}
|
|
235
|
+
const head = split(lines[0]).map((h) => h.replace(/\s*\(.*\)$/, '')) // strip the " (degree_C)" units suffix
|
|
236
|
+
return lines.slice(1).map((l) => {
|
|
237
|
+
const f = split(l)
|
|
238
|
+
return Object.fromEntries(head.map((h, i) => {
|
|
239
|
+
const v = f[i]
|
|
240
|
+
const n = v === '' || v === 'NaN' ? null : Number(v)
|
|
241
|
+
return [h, n === null || Number.isNaN(n) ? v : n]
|
|
242
|
+
}))
|
|
243
|
+
})
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* fetch one slab in the given format. Parquet comes back as bytes for `registerFileBuffer`. `retry`
|
|
248
|
+
* sets the per-attempt timeout and the tries (fetchRetry: 20 s and one retry unless the caller says
|
|
249
|
+
* otherwise: a whole live window can take longer than the one-step map slice).
|
|
250
|
+
*/
|
|
251
|
+
export async function fetchSlab(url: string, format: Format = 'parquet', callback = 'erddapCb', signal?: AbortSignal,
|
|
252
|
+
retry: Omit<RetryOpts, 'signal'> = {}): Promise<Slab> {
|
|
253
|
+
const t0 = (globalThis.performance ?? Date).now()
|
|
254
|
+
if (format === 'jsonp') {
|
|
255
|
+
const data = await fetchJsonp(url, callback, retry.timeoutMs ?? 120_000, signal)
|
|
256
|
+
return { url, format, ms: (globalThis.performance ?? Date).now() - t0, rows: tableToRows(data) }
|
|
257
|
+
}
|
|
258
|
+
return fetchRetry(url, { ...retry, signal }, async (res): Promise<Slab> => {
|
|
259
|
+
if (!res.ok) {
|
|
260
|
+
const msg = (await res.text().catch(() => '')).slice(0, 400)
|
|
261
|
+
throw new Error(`${res.status} ${res.statusText} from ERDDAP: ${msg}`)
|
|
262
|
+
}
|
|
263
|
+
if (format === 'csvp') {
|
|
264
|
+
const text = await res.text()
|
|
265
|
+
return { url, format, ms: (globalThis.performance ?? Date).now() - t0, bytes: text.length, text, rows: parseCsvp(text) }
|
|
266
|
+
}
|
|
267
|
+
// parquet and parquetWMeta alike: raw bytes for registerFileBuffer
|
|
268
|
+
const buffer = new Uint8Array(await res.arrayBuffer())
|
|
269
|
+
return { url, format, ms: (globalThis.performance ?? Date).now() - t0, bytes: buffer.byteLength, buffer }
|
|
270
|
+
})
|
|
271
|
+
}
|
package/src/extent.ts
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// live time extent of an ERDDAP grid dataset, from `<base>/erddap/info/<datasetID>/index.json`.
|
|
2
|
+
//
|
|
3
|
+
// the STAC `cube:dimensions.time.extent` end is null (or stale) for near-real-time grids, and every
|
|
4
|
+
// dataset ends at a different instant: AOML Seascapes is an 8-day product months behind, CRW is a
|
|
5
|
+
// couple of days behind. asking the server means the default window always lands inside the data,
|
|
6
|
+
// instead of 404ing with `"Start" is greater than the axis maximum=…`.
|
|
7
|
+
//
|
|
8
|
+
// the parsing and clamping here are pure functions (see extent.test.ts); only fetchTimeExtent()
|
|
9
|
+
// touches the network.
|
|
10
|
+
import { fetchRetry, type RetryOpts } from './fetchRetry.js'
|
|
11
|
+
|
|
12
|
+
export interface TimeExtent {
|
|
13
|
+
start : string // ISO instant of the first time step, e.g. 2002-08-29T12:00:00Z
|
|
14
|
+
end : string // ISO instant of the last time step
|
|
15
|
+
stepDays ?: number // average spacing in days (1 for daily, 8 for the 8-day product)
|
|
16
|
+
stepLabel?: string // 'daily', '8-day', 'monthly', …
|
|
17
|
+
nValues ?: number
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface InfoTable { table: { columnNames: string[]; rows: unknown[][] } }
|
|
21
|
+
|
|
22
|
+
const isoZ = (secs: number) => new Date(secs * 1000).toISOString().replace(/\.\d{3}Z$/, 'Z')
|
|
23
|
+
/** the UTC calendar day (yyyy-mm-dd) of an ISO instant. */
|
|
24
|
+
export const day = (iso: string) => iso.slice(0, 10)
|
|
25
|
+
const dayMs = 864e5
|
|
26
|
+
/** yyyy-mm-dd, n days after `d` (n may be negative). */
|
|
27
|
+
export function addDays(d: string, n: number): string {
|
|
28
|
+
return new Date(Date.parse(`${day(d)}T00:00:00Z`) + n * dayMs).toISOString().slice(0, 10)
|
|
29
|
+
}
|
|
30
|
+
/** whole days from `a` to `b`, by UTC calendar day. */
|
|
31
|
+
export const daysBetween = (a: string, b: string) =>
|
|
32
|
+
Math.round((Date.parse(`${day(b)}T00:00:00Z`) - Date.parse(`${day(a)}T00:00:00Z`)) / dayMs)
|
|
33
|
+
|
|
34
|
+
/** `8 days 0h 7m 57s` / `1 day` / `12h` -> days. */
|
|
35
|
+
export function parseSpacing(s: string): number | undefined {
|
|
36
|
+
const m = s.match(/([\d.]+)\s*days?/i)
|
|
37
|
+
let days = m ? Number(m[1]) : 0
|
|
38
|
+
const h = s.match(/(\d+)\s*h\b/), mn = s.match(/(\d+)\s*m\b/), sec = s.match(/(\d+)\s*s\b/)
|
|
39
|
+
days += (h ? Number(h[1]) / 24 : 0) + (mn ? Number(mn[1]) / 1440 : 0) + (sec ? Number(sec[1]) / 86400 : 0)
|
|
40
|
+
return days > 0 ? days : undefined
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** a human label for a time step in days. */
|
|
44
|
+
export function stepLabel(days: number | undefined): string | undefined {
|
|
45
|
+
if (!days || !Number.isFinite(days)) return undefined
|
|
46
|
+
const d = Math.round(days)
|
|
47
|
+
if (days < 1) return 'sub-daily'
|
|
48
|
+
if (d === 1) return 'daily'
|
|
49
|
+
if (d === 7) return 'weekly'
|
|
50
|
+
if (d >= 28 && d <= 31) return 'monthly'
|
|
51
|
+
if (d >= 360 && d <= 366) return 'annual'
|
|
52
|
+
return `${d}-day`
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** the ERDDAP `info/<id>/index.json` table -> the dataset's live time extent. */
|
|
56
|
+
export function parseInfoExtent(info: InfoTable, axis = 'time'): TimeExtent | null {
|
|
57
|
+
const rows = (info?.table?.rows ?? []).map((r) => r.map((c) => (c == null ? '' : String(c))))
|
|
58
|
+
const find = (type: string, varName: string, attr: string) =>
|
|
59
|
+
rows.find((r) => r[0] === type && r[1] === varName && r[2] === attr)?.[4]
|
|
60
|
+
|
|
61
|
+
let start: string | undefined, end: string | undefined
|
|
62
|
+
const range = find('attribute', axis, 'actual_range')
|
|
63
|
+
if (range) {
|
|
64
|
+
const [a, b] = range.split(',').map((x) => Number(x.trim()))
|
|
65
|
+
if (Number.isFinite(a) && Number.isFinite(b)) {
|
|
66
|
+
const [lo, hi] = a <= b ? [a, b] : [b, a]
|
|
67
|
+
start = isoZ(lo); end = isoZ(hi)
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
// fall back to the NC_GLOBAL coverage attributes (already ISO strings)
|
|
71
|
+
start ??= find('attribute', 'NC_GLOBAL', 'time_coverage_start')
|
|
72
|
+
end ??= find('attribute', 'NC_GLOBAL', 'time_coverage_end')
|
|
73
|
+
if (!start || !end) return null
|
|
74
|
+
|
|
75
|
+
const dim = rows.find((r) => r[0] === 'dimension' && r[1] === axis)?.[4] ?? ''
|
|
76
|
+
const step = parseSpacing((dim.match(/averageSpacing=([^,]+)/)?.[1] ?? '').trim())
|
|
77
|
+
const nVals = Number(dim.match(/nValues=(\d+)/)?.[1])
|
|
78
|
+
return {
|
|
79
|
+
start, end,
|
|
80
|
+
stepDays : step,
|
|
81
|
+
stepLabel: stepLabel(step),
|
|
82
|
+
nValues : Number.isFinite(nVals) ? nVals : undefined,
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── the date window ───────────────────────────────────────────────────────────
|
|
87
|
+
export interface Window { start: string; end: string } // yyyy-mm-dd, UTC calendar days
|
|
88
|
+
export interface ClampResult extends Window { snapped: boolean; reason?: string }
|
|
89
|
+
|
|
90
|
+
export interface WindowOpts {
|
|
91
|
+
days ?: number // the default span for a daily dataset
|
|
92
|
+
minSteps?: number // at least this many time steps for a coarser dataset
|
|
93
|
+
maxDays ?: number // never span more than this
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** how many days a default window should span for a dataset of this step. */
|
|
97
|
+
export function spanDays(ext: TimeExtent | null, o: WindowOpts = {}): number {
|
|
98
|
+
const { days = 30, minSteps = 8, maxDays = 90 } = o
|
|
99
|
+
const step = ext?.stepDays ?? 1
|
|
100
|
+
return Math.min(maxDays, Math.max(days, step > 1 ? Math.ceil(minSteps * step) : days))
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** the last `days` (or `minSteps` steps) ending at the dataset's last time step. */
|
|
104
|
+
export function defaultWindow(ext: TimeExtent | null, o: WindowOpts = {}): Window {
|
|
105
|
+
const end = ext ? day(ext.end) : addDays(new Date().toISOString(), -2)
|
|
106
|
+
const span = spanDays(ext, o)
|
|
107
|
+
let start = addDays(end, -(span - 1))
|
|
108
|
+
if (ext && daysBetween(ext.start, start) < 0) start = day(ext.start)
|
|
109
|
+
return { start, end }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Keep a user's window inside the dataset's extent.
|
|
114
|
+
* A start after the last time step (or an end before the first) snaps the whole window back to the
|
|
115
|
+
* default one, with a `reason` for the status line, so nothing out-of-range is ever requested.
|
|
116
|
+
*/
|
|
117
|
+
export function clampWindow(w: Window, ext: TimeExtent | null, o: WindowOpts = {}): ClampResult {
|
|
118
|
+
if (!ext) return { ...w, snapped: false }
|
|
119
|
+
const lo = day(ext.start), hi = day(ext.end)
|
|
120
|
+
if (daysBetween(w.start, hi) < 0 || daysBetween(lo, w.end) < 0) {
|
|
121
|
+
const d = defaultWindow(ext, o)
|
|
122
|
+
const why = daysBetween(w.start, hi) < 0
|
|
123
|
+
? `the requested window starts after this dataset ends (${hi})`
|
|
124
|
+
: `the requested window ends before this dataset starts (${lo})`
|
|
125
|
+
return { ...d, snapped: true, reason: `${why}; showing ${d.start} to ${d.end} instead` }
|
|
126
|
+
}
|
|
127
|
+
const start = daysBetween(lo, w.start) < 0 ? lo : w.start
|
|
128
|
+
const end = daysBetween(w.end, hi) < 0 ? hi : w.end
|
|
129
|
+
const snapped = start !== w.start || end !== w.end
|
|
130
|
+
return {
|
|
131
|
+
start, end, snapped,
|
|
132
|
+
reason: snapped ? `clamped to the dataset extent (${lo} to ${hi})` : undefined,
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The ISO instant to put in a griddap time constraint for a calendar day. The extent's own endpoints
|
|
138
|
+
* are used verbatim, so a request never overshoots an axis whose steps are not at 12:00Z (MUR sits
|
|
139
|
+
* at 09:00Z); any interior day falls back to noon.
|
|
140
|
+
*/
|
|
141
|
+
export function timeInstant(date: string, ext: TimeExtent | null, noon: (d: string) => string): string {
|
|
142
|
+
if (ext) {
|
|
143
|
+
if (day(date) === day(ext.end)) return ext.end
|
|
144
|
+
if (day(date) === day(ext.start)) return ext.start
|
|
145
|
+
}
|
|
146
|
+
return noon(date)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// ── fetch ─────────────────────────────────────────────────────────────────────
|
|
150
|
+
const cache = new Map<string, Promise<TimeExtent | null>>()
|
|
151
|
+
const stripSlash = (s: string) => s.replace(/\/+$/, '')
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* `<base>/info/<datasetID>/index.json` -> the live extent, memoised per base+dataset. The fetch is
|
|
155
|
+
* shared by every caller, so no caller's signal is passed to it (a superseded run must not cancel it
|
|
156
|
+
* for the next one); fetchRetry bounds it instead (`INFO_TIMEOUT_MS` per attempt, one retry). null on failure,
|
|
157
|
+
* and a failure is forgotten, so the next call asks again.
|
|
158
|
+
*/
|
|
159
|
+
/** per-attempt timeout of the info request (PacIOOS took 4–9 s on 2026-10-20, and 20 s on 2026-10-09) */
|
|
160
|
+
export const INFO_TIMEOUT_MS = 30_000
|
|
161
|
+
|
|
162
|
+
export function fetchTimeExtent(base: string, datasetId: string, axis = 'time', retry: Omit<RetryOpts, 'signal'> = {}): Promise<TimeExtent | null> {
|
|
163
|
+
const url = `${stripSlash(base)}/info/${datasetId}/index.json`
|
|
164
|
+
const hit = cache.get(url)
|
|
165
|
+
if (hit) return hit
|
|
166
|
+
const p = fetchRetry(url, { timeoutMs: INFO_TIMEOUT_MS, ...retry }, async (res) => {
|
|
167
|
+
if (!res.ok) throw new Error(`${url}: ${res.status} ${res.statusText}`)
|
|
168
|
+
return parseInfoExtent(await res.json(), axis)
|
|
169
|
+
}).catch((e) => { cache.delete(url); console.warn('extent:', e); return null })
|
|
170
|
+
cache.set(url, p)
|
|
171
|
+
return p
|
|
172
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
// fetch with a per-attempt timeout and a retry, for requests to a server that is not ours (ERDDAP).
|
|
2
|
+
//
|
|
3
|
+
// a browser fetch has no timeout of its own: a host that accepts the connection and then stalls holds
|
|
4
|
+
// a run for minutes, and a dropped connection surfaces as the bare "Failed to fetch" (PacIOOS, 2026-10-09:
|
|
5
|
+
// a 20 s info request, then a failed map slice). here every attempt gives up after `timeoutMs`, a network
|
|
6
|
+
// error, a timeout or a 5xx is retried after `backoffMs`, and the last failure is an ErddapUnreachable
|
|
7
|
+
// whose message names the host and what happened, so the toast can say something a person can act on.
|
|
8
|
+
// a 4xx is the server's answer (a bad constraint) and is not retried; the caller's own abort (a newer
|
|
9
|
+
// run) passes straight through as the AbortError it is.
|
|
10
|
+
|
|
11
|
+
export interface RetryOpts {
|
|
12
|
+
signal ?: AbortSignal
|
|
13
|
+
/** give up on one attempt after this long (ms) */
|
|
14
|
+
timeoutMs?: number
|
|
15
|
+
/** tries in all (1 = no retry) */
|
|
16
|
+
attempts ?: number
|
|
17
|
+
/** pause before each retry (ms) */
|
|
18
|
+
backoffMs?: number
|
|
19
|
+
/** for tests */
|
|
20
|
+
fetchFn ?: typeof fetch
|
|
21
|
+
wait ?: (ms: number) => Promise<unknown>
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** the last failure after every attempt: the host did not answer, or answered with a 5xx. */
|
|
25
|
+
export class ErddapUnreachable extends Error {
|
|
26
|
+
constructor(readonly host: string, readonly reason: string, readonly attempts: number, readonly url: string) {
|
|
27
|
+
super(`${host} did not answer (${reason}${attempts > 1 ? `, ${attempts} tries` : ''})`)
|
|
28
|
+
this.name = 'ErddapUnreachable'
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
|
33
|
+
export const hostOf = (url: string) => { try { return new URL(url).host } catch { return url } }
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* fetch `url` and `read` the response (the default returns it unread), inside the same timeout: a
|
|
37
|
+
* stall while the body downloads is a timeout too. A network error (a TypeError, the browser's
|
|
38
|
+
* "Failed to fetch"), a timeout or a 5xx is retried; a 4xx goes to `read` (which may throw the
|
|
39
|
+
* server's message), and whatever `read` throws for its own reasons is not retried.
|
|
40
|
+
*/
|
|
41
|
+
export async function fetchRetry<T = Response>(url: string, o: RetryOpts = {},
|
|
42
|
+
read: (res: Response) => Promise<T> = async (r) => r as T): Promise<T> {
|
|
43
|
+
const { signal, timeoutMs = 20_000, attempts = 2, backoffMs = 1_500, fetchFn = fetch, wait = sleep } = o
|
|
44
|
+
let reason = ''
|
|
45
|
+
for (let i = 0; i < attempts; i++) {
|
|
46
|
+
if (i > 0) await wait(backoffMs)
|
|
47
|
+
if (signal?.aborted) throw signal.reason ?? new DOMException('aborted', 'AbortError')
|
|
48
|
+
const timer = AbortSignal.timeout(timeoutMs)
|
|
49
|
+
const sig = signal ? AbortSignal.any([signal, timer]) : timer
|
|
50
|
+
try {
|
|
51
|
+
const res = await fetchFn(url, { signal: sig })
|
|
52
|
+
if (res.status >= 500) { reason = `HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`; continue }
|
|
53
|
+
return await read(res)
|
|
54
|
+
} catch (e) {
|
|
55
|
+
if (signal?.aborted) throw e // superseded: not a failure
|
|
56
|
+
if (timer.aborted) reason = `no response within ${Math.round(timeoutMs / 1000)} s`
|
|
57
|
+
else if (e instanceof TypeError) reason = `network error: ${e.message}`
|
|
58
|
+
else throw e // read()'s own error, e.g. a 4xx message
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
throw new ErddapUnreachable(hostOf(url), reason, attempts, url)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* The toast's words when the live map step fails after the precomputed statistics are on screen:
|
|
66
|
+
* which host, what happened, and that the statistics are not affected.
|
|
67
|
+
*/
|
|
68
|
+
export function mapFailMessage(e: unknown): string {
|
|
69
|
+
const why = e instanceof Error ? e.message : String(e)
|
|
70
|
+
return `The map's latest time step did not load: ${why}. The precomputed statistics are unaffected; retry the map from the Time strip.`
|
|
71
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// @oceanmetrics/engine: the browser engine of Ocean Metrics. the mask, the lobes, the ERDDAP requests, the SQL templates
|
|
2
|
+
// shared with the Python engine (om_extract), DuckDB-WASM as the statistics engine, the chunked live preview and the
|
|
3
|
+
// map's grid slice. see the README for what moved from where.
|
|
4
|
+
export { axisEdges, countVertices, gridMask, pointMask, polygonParts, reduceRing, reduceRings } from './mask.js'
|
|
5
|
+
export type { GeoInput, MaskCell, MaskMethod, MaskOptions, MaskPoint, MaskResult, PointMaskResult } from './mask.js'
|
|
6
|
+
export { polygonCoordinates, wkbToGeoJSON } from './wkb.js'
|
|
7
|
+
export { placeLobes } from './lobes.js'
|
|
8
|
+
export type { Lobe, LobePlace } from './lobes.js'
|
|
9
|
+
export {
|
|
10
|
+
AXIS_TIMEOUT_MS, SLICE_TIMEOUT_MS, constraint, fetchAxis, fetchJsonp, fetchSlab, griddapSliceUrl, griddapUrl, griddapUrls,
|
|
11
|
+
isParquet, noonZ, parseCsvp, pickFormat, tableConstraint, tableToRows, tabledapPlaceConstraints, tabledapUrl,
|
|
12
|
+
} from './erddap.js'
|
|
13
|
+
export type { Collection, ErddapMeta, Format, GriddapSpec, Slab, TableConstraint, TableOp, TabledapSpec } from './erddap.js'
|
|
14
|
+
export {
|
|
15
|
+
INFO_TIMEOUT_MS, addDays, clampWindow, day, daysBetween, defaultWindow, fetchTimeExtent, parseInfoExtent, parseSpacing,
|
|
16
|
+
spanDays, stepLabel, timeInstant,
|
|
17
|
+
} from './extent.js'
|
|
18
|
+
export type { ClampResult, InfoTable, TimeExtent, Window, WindowOpts } from './extent.js'
|
|
19
|
+
export { ErddapUnreachable, fetchRetry, hostOf, mapFailMessage } from './fetchRetry.js'
|
|
20
|
+
export type { RetryOpts } from './fetchRetry.js'
|
|
21
|
+
export { axisSpacing, cellPoints, cellSquares, indexBounds, normalizeLon, pickCenter, placeMapBounds } from './cells.js'
|
|
22
|
+
export type { CellPoints, CellProps, CellSquares, CellSquaresOptions, LonCenter, ValueCell } from './cells.js'
|
|
23
|
+
export {
|
|
24
|
+
isCategorical, lobeLonSpan, resolveDataset, statsTemplate, toDataset, toDatasetLon, toLongFormat, valueExpr, valueLabel,
|
|
25
|
+
} from './dataset.js'
|
|
26
|
+
export type { CubeVariable, Dataset, LongFormat } from './dataset.js'
|
|
27
|
+
export { MAX_CHUNK_DAYS, MAX_REQUEST_VALUES, chunkDaysFor, chunks, window } from './chunks.js'
|
|
28
|
+
export type { ChunkLimits } from './chunks.js'
|
|
29
|
+
export { lit, render, template, templateNames } from './sql.js'
|
|
30
|
+
export type { Param, Params, TemplateName } from './sql.js'
|
|
31
|
+
export { TEMPLATES } from './sqlTemplates.js'
|
|
32
|
+
export { Engine, connectBrowser, toRows } from './engine.js'
|
|
33
|
+
export type { Duck, DuckConnLike, DuckDbLike, EngineEvent, Row } from './engine.js'
|
|
34
|
+
export { isoDay, lobeAxes, lobeMasks, previewWindow, runPreview, sliceSource } from './preview.js'
|
|
35
|
+
export type {
|
|
36
|
+
LobeAxes, PreviewChunk, PreviewDataset, PreviewOptions, PreviewResult, SliceCell, SliceSource, SliceSourceOptions,
|
|
37
|
+
} from './preview.js'
|
package/src/lobes.ts
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// the lobe split of a place: one griddap request per lobe. moved from erddap-places `app/src/lib/gazetteer.ts`
|
|
2
|
+
// (`placeLobes`, `Lobe`, `partBbox`); the logic is unchanged (the Python twin `om_extract.geom.place_lobes` and the
|
|
3
|
+
// parity fixtures in extract/tests/fixtures/mask were generated from it). `LobePlace` is the slice of the gazetteer's
|
|
4
|
+
// `Place` the split reads.
|
|
5
|
+
import { polygonCoordinates } from './wkb.js'
|
|
6
|
+
import type { Feature, FeatureCollection, Geometry, Polygon } from 'geojson'
|
|
7
|
+
|
|
8
|
+
/** what placeLobes() reads of a place: its id and its (Multi)Polygon, already split at ±180 */
|
|
9
|
+
export interface LobePlace { place_id: string; geometry: Geometry }
|
|
10
|
+
// the name the moved function text uses (kept, so its text stays byte-identical to the original)
|
|
11
|
+
type Place = LobePlace
|
|
12
|
+
|
|
13
|
+
/** one griddap request's worth of a place: its own bbox and the polygons inside it. */
|
|
14
|
+
export interface Lobe {
|
|
15
|
+
id : string
|
|
16
|
+
bbox : [number, number, number, number]
|
|
17
|
+
geojson : FeatureCollection
|
|
18
|
+
nParts : number
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const partBbox = (rings: number[][][]): [number, number, number, number] => {
|
|
22
|
+
let x0 = Infinity, y0 = Infinity, x1 = -Infinity, y1 = -Infinity
|
|
23
|
+
for (const [x, y] of rings[0]) {
|
|
24
|
+
if (x < x0) x0 = x; if (x > x1) x1 = x
|
|
25
|
+
if (y < y0) y0 = y; if (y > y1) y1 = y
|
|
26
|
+
}
|
|
27
|
+
return [x0, y0, x1, y1]
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Split a place into the lobes that each get one griddap request.
|
|
32
|
+
*
|
|
33
|
+
* Parts are grouped by side of the antimeridian, and only when the place actually spans it (a
|
|
34
|
+
* lon extent wider than 180°, as for PMNM): a place wholly on one side stays a single request
|
|
35
|
+
* covering all its parts. The polygon parts themselves are never modified — the gazetteer has
|
|
36
|
+
* already split them at ±180.
|
|
37
|
+
*/
|
|
38
|
+
export function placeLobes(place: Place): Lobe[] {
|
|
39
|
+
const parts = polygonCoordinates(place.geometry)
|
|
40
|
+
if (!parts.length) return []
|
|
41
|
+
const boxes = parts.map(partBbox)
|
|
42
|
+
const x0 = Math.min(...boxes.map((b) => b[0])), x1 = Math.max(...boxes.map((b) => b[2]))
|
|
43
|
+
const split = x1 - x0 > 180
|
|
44
|
+
|
|
45
|
+
const groups = new Map<string, number[]>()
|
|
46
|
+
boxes.forEach((b, i) => {
|
|
47
|
+
const g = !split ? 'all' : (b[0] + b[2]) / 2 < 0 ? 'W' : 'E'
|
|
48
|
+
;(groups.get(g) ?? groups.set(g, []).get(g)!).push(i)
|
|
49
|
+
})
|
|
50
|
+
|
|
51
|
+
const lobes: Lobe[] = []
|
|
52
|
+
for (const [g, idx] of groups) {
|
|
53
|
+
const feats: Feature<Polygon>[] = idx.map((i) => ({
|
|
54
|
+
type: 'Feature', properties: { place_id: place.place_id },
|
|
55
|
+
geometry: { type: 'Polygon', coordinates: parts[i] },
|
|
56
|
+
}))
|
|
57
|
+
lobes.push({
|
|
58
|
+
id : `${place.place_id}${g === 'all' ? '' : `:${g}`}`,
|
|
59
|
+
bbox : [Math.min(...idx.map((i) => boxes[i][0])), Math.min(...idx.map((i) => boxes[i][1])),
|
|
60
|
+
Math.max(...idx.map((i) => boxes[i][2])), Math.max(...idx.map((i) => boxes[i][3]))],
|
|
61
|
+
geojson: { type: 'FeatureCollection', features: feats },
|
|
62
|
+
nParts : idx.length,
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
return lobes.sort((a, b) => a.bbox[0] - b.bbox[0])
|
|
66
|
+
}
|