@mnmal-ai/substrate 1.0.0-alpha.15 → 1.0.0-alpha.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/dist/{boot-store-CvQabBdk.js → boot-store-DNU0UfX7.js} +19 -66
- package/dist/{bridge-CvZGzNIO.d.ts → bridge-5VHZiH9o.d.ts} +1 -1
- package/dist/{cli-6evrDG1o.js → cli-CwWUYUMv.js} +71 -13
- package/dist/client.d.ts +3 -3
- package/dist/client.js +3 -3
- package/dist/{errors-BXxm9eZF.d.ts → errors-DHgT7Fa8.d.ts} +8 -2
- package/dist/{index-EN-nQsX8.d.ts → index-DKJXQ031.d.ts} +1 -1
- package/dist/{node-version-C5FSmwJX.js → node-version-CxqcqECU.js} +1 -1
- package/dist/pack.d.ts +18 -2
- package/dist/pack.js +2 -2
- package/dist/{pg-edges-store-DHbpr8QY.js → pg-edges-store-CSJ7sG2l.js} +1 -1
- package/dist/{plugin-Dpa_Y7Ep.js → plugin-Dk9RC3LH.js} +2 -2
- package/dist/proposals-bootstrap.js +2 -2
- package/dist/proposals.d.ts +5 -3
- package/dist/proposals.js +1 -1
- package/dist/recall-bootstrap.js +2 -2
- package/dist/recall.d.ts +1 -1
- package/dist/recall.js +2 -2
- package/dist/{repl-DWXNLUyd.js → repl-BNnKFG3y.js} +1 -1
- package/dist/{row-Dh57ShWL.d.ts → row-DHxnCnhA.d.ts} +10 -8
- package/dist/{run-BTdS6Z6i.js → run-NmwNmVUD.js} +142 -106
- package/dist/{schema-B--_bHjn.d.ts → schema-B5WBO8yh.d.ts} +79 -19
- package/dist/sdk.d.ts +48 -6
- package/dist/sdk.js +4 -4
- package/dist/{src-EqhJm93D.js → src-7lp1y4HK.js} +3 -3
- package/dist/{src-BMHI_rTJ.js → src-BHh6UZ7L.js} +2141 -493
- package/dist/{src-DWqzrm3O.js → src-CdbUzLhB.js} +209 -36
- package/dist/{src-Bi3NhEsT.js → src-DR_hUj2p.js} +1 -1
- package/dist/{src-CHND7-Pm.js → src-DV6PcXym.js} +2 -2
- package/dist/{src-CZQ25H4i.js → src-DWrFdHrY.js} +1018 -252
- package/dist/{src-B4sqnLRy.js → src-DwynOVMk.js} +77 -28
- package/dist/substrate-admin.js +2 -2
- package/dist/substrate.js +2 -2
- package/dist/testing.d.ts +5 -3
- package/dist/testing.js +4 -4
- package/dist/{ws-CpHWxwp9.d.ts → ws-BL9nSELO.d.ts} +2 -2
- package/package.json +23 -23
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { $t as METADATA_FILTER_FIELDS, At as encodeCursor, Ct as walkAggExprType, Dt as endsWithEscape, Et as LIKE_TRAILING_ESCAPE, In as assertNoRepeatedIds, Mt as fullBuiltinOperators, Nt as applyExtensionResultMeta, On as WALK_UNSUPPORTED_REF, Ot as CursorDecodeError, Pt as fieldRequested, Sn as VALUE_OUT_OF_RANGE, St as metadataExprType, Tt as isJsonPathPredicate, _n as ReservedField, at as forEachWalkHop, bn as STORE_BUSY, ct as preResolveExtensionOperatorsDetailed, dt as canRaise, ft as isDateField, gt as refNames, ht as isRefOperand, jt as normalizeOrderBy, kn as filterTooBroad, kt as decodeCursor, lt as aggParts, mn as QUERY_TIMEOUT, mt as groupConjuncts, on as normalizeIsoTimestamp, ot as sharedPicks, pn as NamespaceMismatch, pt as bindingWalks, rn as createKeyedMutex, st as windowGroupBy, ut as aggType, vn as ResolverFrameError, wt as assertJsonPathPredicate, xn as UnsupportedFilterError, xt as exprTypeOfField } from "./src-BHh6UZ7L.js";
|
|
3
|
+
import { A as parseInt64, M as stringifyJsonExact, N as DATE_UNITS, O as isInt16, T as naturalOrderOf, j as stringifyJson, k as isInt32, w as isColumnField } from "./src-CdbUzLhB.js";
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
5
|
|
|
6
6
|
//#region ../../packages/substrate-dataset-pg/src/advisory-lock.ts
|
|
@@ -163,7 +163,8 @@ function encodeRow(table, row) {
|
|
|
163
163
|
const written = value instanceof Date ? value.toJSON() : value;
|
|
164
164
|
const v = toColumn(held, written);
|
|
165
165
|
out.columns[held.column] = v;
|
|
166
|
-
|
|
166
|
+
const back = v === null ? void 0 : readBack(held.kind, v);
|
|
167
|
+
if (v === null || (held.kind === "int64" ? back !== parseInt64(written) : back !== written)) out.raw[field] = written;
|
|
167
168
|
}
|
|
168
169
|
return out;
|
|
169
170
|
}
|
|
@@ -202,6 +203,12 @@ function toColumn(column, value) {
|
|
|
202
203
|
if (typeof value === "number") return String(value);
|
|
203
204
|
return typeof value === "string" ? value : null;
|
|
204
205
|
case "number": return typeof value === "number" ? value : null;
|
|
206
|
+
case "int16": return isInt16(value) ? value : null;
|
|
207
|
+
case "int32": return isInt32(value) ? value : null;
|
|
208
|
+
case "int64": {
|
|
209
|
+
const n = parseInt64(value);
|
|
210
|
+
return n === void 0 ? null : String(n);
|
|
211
|
+
}
|
|
205
212
|
case "boolean": return typeof value === "boolean" ? value : null;
|
|
206
213
|
case "date": {
|
|
207
214
|
if (typeof value !== "string") return null;
|
|
@@ -212,18 +219,34 @@ function toColumn(column, value) {
|
|
|
212
219
|
}
|
|
213
220
|
/** What reading column value `v` back gives, or `undefined` when that read is lossy. */
|
|
214
221
|
function readBack(kind, v) {
|
|
222
|
+
if (kind === "int64") return BigInt(v);
|
|
215
223
|
if (kind !== "date") return v;
|
|
216
224
|
const d = new Date(v);
|
|
217
225
|
return d.getUTCFullYear() <= 100 ? void 0 : d.toISOString();
|
|
218
226
|
}
|
|
227
|
+
/** `utcTextSql(col, 'US')` output for a year 1 to 9999 instant. */
|
|
228
|
+
const UTC_US_TEXT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/;
|
|
219
229
|
/** A column value as pg returns it, in the JSON form a row carries: dates as `toISOString()`. */
|
|
220
230
|
function canonical(kind, value) {
|
|
231
|
+
if (kind === "int64") return BigInt(value);
|
|
221
232
|
if (kind !== "date") return value;
|
|
233
|
+
if (typeof value === "string" && UTC_US_TEXT.test(value)) return `${value.slice(0, 23)}Z`;
|
|
222
234
|
return (value instanceof Date ? value : new Date(String(value))).toISOString();
|
|
223
235
|
}
|
|
224
236
|
|
|
225
237
|
//#endregion
|
|
226
238
|
//#region ../../packages/substrate-dataset-pg/src/pg-sql/primitives.ts
|
|
239
|
+
/**
|
|
240
|
+
* Shared foundations for the pg SQL builders: the Fragment alias, low-level
|
|
241
|
+
* primitives, promoted-column machinery and scalar helpers. Imports nothing
|
|
242
|
+
* from the other builders and holds no connection state.
|
|
243
|
+
*
|
|
244
|
+
* SQL-injection contract: every caller-supplied
|
|
245
|
+
* value flows through postgres.js's parameterised tagged-template slots.
|
|
246
|
+
* `sql.unsafe(...)` is used ONLY for identifiers/cast suffixes drawn from
|
|
247
|
+
* closed constant maps in this file, and for field names inlined as escaped
|
|
248
|
+
* literals by `sqlStringLiteral`.
|
|
249
|
+
*/
|
|
227
250
|
const idToText = (id) => typeof id === "number" ? String(id) : id;
|
|
228
251
|
/**
|
|
229
252
|
* Row-metadata field → native column. Predicates on these fields use the
|
|
@@ -307,6 +330,11 @@ const jsonSql = (path, alias, jsonb = "extra") => `${qualifier(alias)}${jsonb}${
|
|
|
307
330
|
const rawTextSql = (path, alias, jsonb) => `(${jsonSql(path.slice(0, -1), alias, jsonb)} ->> ${sqlStringLiteral(path[path.length - 1])})`;
|
|
308
331
|
const textSql = (path, alias, jsonb) => `(${rawTextSql(path, alias, jsonb)} COLLATE "C")`;
|
|
309
332
|
const numSql = (path, alias, jsonb) => `substrate_num(${jsonSql(path, alias, jsonb)})`;
|
|
333
|
+
/** The stored number as a bigint, or NULL for anything that is not an int64 (never raises). */
|
|
334
|
+
const int64Sql = (path, alias, jsonb) => {
|
|
335
|
+
const n = numSql(path, alias, jsonb);
|
|
336
|
+
return `(CASE WHEN ${n} = trunc(${n}) AND ${n} BETWEEN -9223372036854775808 AND 9223372036854775807 THEN ${n}::bigint END)`;
|
|
337
|
+
};
|
|
310
338
|
const tsSql = (path, alias, jsonb) => `substrate_ts(${jsonSql(path, alias, jsonb)})`;
|
|
311
339
|
/** The jsonb-path expression for `as`: the one every body/extra builder below emits. */
|
|
312
340
|
function pathSql(path, as, alias, jsonb) {
|
|
@@ -314,6 +342,7 @@ function pathSql(path, as, alias, jsonb) {
|
|
|
314
342
|
case "text": return textSql(path, alias, jsonb);
|
|
315
343
|
case "rawText": return rawTextSql(path, alias, jsonb);
|
|
316
344
|
case "number": return numSql(path, alias, jsonb);
|
|
345
|
+
case "int64": return int64Sql(path, alias, jsonb);
|
|
317
346
|
case "date": return tsSql(path, alias, jsonb);
|
|
318
347
|
case "boolean": return `${rawTextSql(path, alias, jsonb)}::boolean`;
|
|
319
348
|
case "json": return `(${jsonSql(path, alias, jsonb)})`;
|
|
@@ -321,7 +350,8 @@ function pathSql(path, as, alias, jsonb) {
|
|
|
321
350
|
}
|
|
322
351
|
/** The `as` a column of `kind` compares and sorts as. */
|
|
323
352
|
function columnAs(kind) {
|
|
324
|
-
|
|
353
|
+
if (kind === "int16" || kind === "int32") return "int64";
|
|
354
|
+
return kind === "number" || kind === "int64" || kind === "date" || kind === "boolean" ? kind : "text";
|
|
325
355
|
}
|
|
326
356
|
/**
|
|
327
357
|
* A date column as text in the `toISOString()` grammar with microseconds, read
|
|
@@ -418,6 +448,10 @@ function bodyJson(sql, path, alias, jsonb) {
|
|
|
418
448
|
function bodyNum(sql, path, alias, jsonb) {
|
|
419
449
|
return sql.unsafe(pathSql(path, "number", alias, jsonb));
|
|
420
450
|
}
|
|
451
|
+
/** The stored number as a bigint, NULL unless it is an int64. */
|
|
452
|
+
function bodyInt64(sql, path, alias, jsonb) {
|
|
453
|
+
return sql.unsafe(pathSql(path, "int64", alias, jsonb));
|
|
454
|
+
}
|
|
421
455
|
/** `substrate_ts(body -> …)`: the instant, or NULL for anything but an ISO date or date-time string. */
|
|
422
456
|
function bodyTs(sql, path, alias, jsonb) {
|
|
423
457
|
return sql.unsafe(pathSql(path, "date", alias, jsonb));
|
|
@@ -427,6 +461,15 @@ function indexExpressionSql(kind, field, alias, jsonb) {
|
|
|
427
461
|
return pathSql([field], kind, alias, jsonb);
|
|
428
462
|
}
|
|
429
463
|
/**
|
|
464
|
+
* `like` / `notLike` on a text expression: case-sensitive, `\` escapes, and never
|
|
465
|
+
* NULL (a NULL value matches neither), as the memory evaluator's.
|
|
466
|
+
*/
|
|
467
|
+
function likeFragment(sql, expr, field, op, operand) {
|
|
468
|
+
if (typeof operand !== "string") throw new UnsupportedFilterError(field, op, "requires a string value");
|
|
469
|
+
if (endsWithEscape(operand)) throw new UnsupportedFilterError(field, op, LIKE_TRAILING_ESCAPE);
|
|
470
|
+
return op === "like" ? sql`COALESCE((${expr} LIKE ${operand} ESCAPE '\\'), FALSE)` : sql`COALESCE((${expr} NOT LIKE ${operand} ESCAPE '\\'), FALSE)`;
|
|
471
|
+
}
|
|
472
|
+
/**
|
|
430
473
|
* `cond` where `expr` is not NULL, else FALSE, never NULL, so a caller's NOT
|
|
431
474
|
* stays sound. The bare `cond` conjunct keeps it index-servable, where a
|
|
432
475
|
* COALESCE would hide it from the planner.
|
|
@@ -437,6 +480,7 @@ function notNullAnd(sql, expr, cond) {
|
|
|
437
480
|
const asJsonValue = (v) => v;
|
|
438
481
|
function scalarParam(value) {
|
|
439
482
|
if (value instanceof Date) return value.toISOString();
|
|
483
|
+
if (typeof value === "bigint") return value.toString();
|
|
440
484
|
if (value === null || typeof value === "string" || typeof value === "number" || typeof value === "boolean") return value;
|
|
441
485
|
return String(value);
|
|
442
486
|
}
|
|
@@ -482,13 +526,17 @@ function interleaveOperator(sql, fragments, op) {
|
|
|
482
526
|
//#endregion
|
|
483
527
|
//#region ../../packages/substrate-dataset-pg/src/pg-sql/projection.ts
|
|
484
528
|
/**
|
|
485
|
-
*
|
|
486
|
-
*
|
|
529
|
+
* Read-path shaping: which body keys to materialise, and how the native
|
|
530
|
+
* metadata columns fold back into the `RowMetadata` shape callers see.
|
|
531
|
+
*/
|
|
532
|
+
/**
|
|
533
|
+
* The SELECT items carrying a read's row, then `walk` (its walk selections'
|
|
534
|
+
* columns, from `buildWalkColumns`). The caller selects the metadata columns.
|
|
487
535
|
*
|
|
488
536
|
* For `decodeRow`: each column (of a requested field, when
|
|
489
537
|
* `fields` is given), `orphaned` ones included (F35), a date column as UTC
|
|
490
538
|
* text; then `extra AS extra`, whole, or its requested keys with `id` and
|
|
491
|
-
* `$raw` kept.
|
|
539
|
+
* `$raw` kept.
|
|
492
540
|
*
|
|
493
541
|
* Keys are bound as parameters, never concatenated.
|
|
494
542
|
*/
|
|
@@ -496,15 +544,20 @@ function buildRowProjection(sql, table, fields, alias = "r", walk = null) {
|
|
|
496
544
|
const q = qualifier(alias);
|
|
497
545
|
const keys = /* @__PURE__ */ new Set(["id"]);
|
|
498
546
|
for (const key of Object.keys(fields ?? {})) if (!NON_BODY_FIELD_KEYS.has(key)) keys.add(key);
|
|
499
|
-
const merged = (doc) => walk === null ? doc : sql`(${doc} || ${walk})`;
|
|
500
547
|
const columns = table.columns().filter((c) => fields === void 0 || Object.hasOwn(fields, c.field)).map((c) => {
|
|
501
548
|
const col = columnSql(c, alias);
|
|
502
549
|
return c.kind === "date" ? `${utcTextSql(col, "US")} AS ${c.column}` : col;
|
|
503
550
|
});
|
|
504
|
-
const
|
|
551
|
+
const ex = sql.unsafe(`${q}extra`);
|
|
552
|
+
const wanted = [...keys, RAW_KEY];
|
|
553
|
+
const subset = sql`COALESCE(CASE WHEN pg_column_size(${ex}) < ${sql.unsafe(String(INLINE_EXTRA_BYTES))} THEN (SELECT jsonb_object_agg(k, ${ex} -> k) FROM unnest(${wanted}::text[]) k WHERE ${ex} ? k) ELSE (SELECT jsonb_object_agg(e.key, e.value) FROM jsonb_each(${ex}) e WHERE e.key = ANY(${wanted}::text[])) END, '{}'::jsonb)`;
|
|
554
|
+
const allColumns = [...keys].every((k) => k === "id" || table.column(k)?.state === "column");
|
|
555
|
+
const extra = fields === void 0 ? ex : allColumns ? sql`CASE WHEN ${ex} ?| ${["id", RAW_KEY]}::text[] THEN ${subset} ELSE '{}'::jsonb END` : subset;
|
|
505
556
|
const head = columns.length === 0 ? "" : `${columns.join(", ")}, `;
|
|
506
|
-
return sql`${sql.unsafe(head)}${
|
|
557
|
+
return sql`${sql.unsafe(head)}${extra} AS extra${walk === null ? sql`` : sql`, ${walk}`}`;
|
|
507
558
|
}
|
|
559
|
+
/** Below pg's TOAST threshold (~2 kB), a jsonb value is stored inline and uncompressed. */
|
|
560
|
+
const INLINE_EXTRA_BYTES = 2e3;
|
|
508
561
|
/**
|
|
509
562
|
* A jsonb object from `'k'::text, <value>` pairs (the output key needs
|
|
510
563
|
* `::text`: jsonb_build_object's VARIADIC any cannot infer a parameter's
|
|
@@ -526,6 +579,21 @@ function join(parts, sep) {
|
|
|
526
579
|
}
|
|
527
580
|
return out;
|
|
528
581
|
}
|
|
582
|
+
/** Projection keys that read row metadata: `_metadata`, the author names, and the metadata columns. */
|
|
583
|
+
const METADATA_KEYS = /* @__PURE__ */ new Set([
|
|
584
|
+
"_metadata",
|
|
585
|
+
"createdBy",
|
|
586
|
+
"updatedBy",
|
|
587
|
+
...Object.keys(METADATA_FILTER_FIELDS).filter((k) => k !== "id")
|
|
588
|
+
]);
|
|
589
|
+
/** True unless `fields` lists its keys and none of them reads row metadata. */
|
|
590
|
+
function readsRowMetadata(fields) {
|
|
591
|
+
return fields === void 0 || Object.keys(fields).some((k) => METADATA_KEYS.has(k));
|
|
592
|
+
}
|
|
593
|
+
/** True unless `fields` lists its keys without `shortId`. */
|
|
594
|
+
function readsShortId(fields) {
|
|
595
|
+
return fields === void 0 || Object.hasOwn(fields, "shortId");
|
|
596
|
+
}
|
|
529
597
|
function assembleRowMetadata(record) {
|
|
530
598
|
const seq = record.seq == null ? void 0 : Number(record.seq);
|
|
531
599
|
if (seq === void 0 && record.created_at === null && record.updated_at === null && record.created_by_kind === null && record.created_by_id === null && record.updated_by_kind === null && record.updated_by_id === null) return;
|
|
@@ -548,12 +616,29 @@ function assembleRowMetadata(record) {
|
|
|
548
616
|
|
|
549
617
|
//#endregion
|
|
550
618
|
//#region ../../packages/substrate-dataset-pg/src/write-rows.ts
|
|
619
|
+
/**
|
|
620
|
+
* The `tables` layout's row writes: one statement for any number of rows.
|
|
621
|
+
* `PgDataset.write` upserts through {@link writeRowsStatement}, and its patch
|
|
622
|
+
* paths merge through {@link inputCte} and {@link mergeAssignments}, so every
|
|
623
|
+
* write applies `applyPatch`'s rule (row-codec.ts) in SQL:
|
|
624
|
+
*
|
|
625
|
+
* - a column the row writes takes its value (NULL for a cleared field), and
|
|
626
|
+
* one it does not write keeps the stored value;
|
|
627
|
+
* - `extra` loses every key the row writes (`unset`) and the old `$raw`, then
|
|
628
|
+
* takes the row's `extra`; `$raw` loses the written keys, takes the row's
|
|
629
|
+
* `$raw`, and is dropped when it ends empty.
|
|
630
|
+
*
|
|
631
|
+
* So no declared column key is ever left in `extra` by a write (F13).
|
|
632
|
+
*/
|
|
551
633
|
/** The SQL type an input column is cast to; every value is bound as text. */
|
|
552
634
|
const CAST = {
|
|
553
635
|
text: "text",
|
|
554
636
|
enum: "text",
|
|
555
637
|
crossRef: "text",
|
|
556
638
|
number: "float8",
|
|
639
|
+
int16: "int2",
|
|
640
|
+
int32: "int4",
|
|
641
|
+
int64: "bigint",
|
|
557
642
|
date: "timestamptz",
|
|
558
643
|
boolean: "boolean"
|
|
559
644
|
};
|
|
@@ -646,8 +731,8 @@ function inputCte(sql, table, rows, metadata, stored) {
|
|
|
646
731
|
const cols = writableColumns(table, rows);
|
|
647
732
|
const arrays = [
|
|
648
733
|
rows.map((r) => r.id),
|
|
649
|
-
rows.map((r) =>
|
|
650
|
-
rows.map((r) =>
|
|
734
|
+
rows.map((r) => stringifyJsonExact(r.extra)),
|
|
735
|
+
rows.map((r) => stringifyJsonExact(r.raw)),
|
|
651
736
|
rows.map((r) => JSON.stringify(r.unset)),
|
|
652
737
|
...cols.map((c) => rows.map((r) => {
|
|
653
738
|
const v = Object.hasOwn(r.columns, c.column) ? r.columns[c.column] : null;
|
|
@@ -1113,7 +1198,7 @@ const SAFE_CASTS = [`CREATE OR REPLACE FUNCTION substrate_num(v jsonb) RETURNS p
|
|
|
1113
1198
|
THEN v OPERATOR(pg_catalog.#>>) '{}' END,
|
|
1114
1199
|
'${String.raw`^([0-9]{4}-[0-9]{2}-[0-9]{2})(?:[T ]((?:[01][0-9]|2[0-3]):[0-5][0-9])(?::([0-5][0-9])(?:\.([0-9]{1,6})[0-9]{0,3})?)?)?(Z|[+-](?:0[0-9]|1[0-5])(?::?[0-5][0-9])?)?$`}') AS m
|
|
1115
1200
|
) AS n $$`];
|
|
1116
|
-
const TX_STATEMENTS$
|
|
1201
|
+
const TX_STATEMENTS$6 = [
|
|
1117
1202
|
`CREATE TABLE IF NOT EXISTS row_ids (
|
|
1118
1203
|
namespace text NOT NULL,
|
|
1119
1204
|
type text NOT NULL,
|
|
@@ -1230,7 +1315,7 @@ const uuidCheck = (table, column) => `DO $$ BEGIN
|
|
|
1230
1315
|
CHECK (${column} ~ '${UUID_REGEX}');
|
|
1231
1316
|
EXCEPTION WHEN duplicate_object THEN NULL;
|
|
1232
1317
|
END $$`;
|
|
1233
|
-
const TX_STATEMENTS$
|
|
1318
|
+
const TX_STATEMENTS$5 = [
|
|
1234
1319
|
uuidCheck("row_ids", "id"),
|
|
1235
1320
|
uuidCheck("short_ids", "row_id"),
|
|
1236
1321
|
uuidCheck("edges", "from_id"),
|
|
@@ -1256,7 +1341,7 @@ const TX_STATEMENTS$2 = [
|
|
|
1256
1341
|
* through a stale descriptor is retried rather than silently misplaced.
|
|
1257
1342
|
* A constant default: no table rewrite.
|
|
1258
1343
|
*/
|
|
1259
|
-
const TX_STATEMENTS$
|
|
1344
|
+
const TX_STATEMENTS$4 = ["ALTER TABLE substrate_type_tables ADD COLUMN IF NOT EXISTS version bigint NOT NULL DEFAULT 0"];
|
|
1260
1345
|
|
|
1261
1346
|
//#endregion
|
|
1262
1347
|
//#region ../../packages/substrate-dataset-pg/src/migrations/0004t_field_error_version.ts
|
|
@@ -1268,7 +1353,34 @@ const TX_STATEMENTS$1 = ["ALTER TABLE substrate_type_tables ADD COLUMN IF NOT EX
|
|
|
1268
1353
|
* than rescanning the whole table every sweep for a conflict that has not
|
|
1269
1354
|
* changed (promote.ts). Nullable, no default: no table rewrite.
|
|
1270
1355
|
*/
|
|
1271
|
-
const TX_STATEMENTS = ["ALTER TABLE substrate_type_fields ADD COLUMN IF NOT EXISTS error_version bigint NULL"];
|
|
1356
|
+
const TX_STATEMENTS$3 = ["ALTER TABLE substrate_type_fields ADD COLUMN IF NOT EXISTS error_version bigint NULL"];
|
|
1357
|
+
|
|
1358
|
+
//#endregion
|
|
1359
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0005t_int64_field_kind.ts
|
|
1360
|
+
/**
|
|
1361
|
+
* 0005t_int64_field_kind — `int64` joins the field kinds the type catalog
|
|
1362
|
+
* accepts (a `bigint` column).
|
|
1363
|
+
*/
|
|
1364
|
+
const TX_STATEMENTS$2 = ["ALTER TABLE substrate_type_fields DROP CONSTRAINT IF EXISTS substrate_type_fields_kind_check", `ALTER TABLE substrate_type_fields ADD CONSTRAINT substrate_type_fields_kind_check
|
|
1365
|
+
CHECK (kind IN ('text','enum','number','int64','date','boolean','crossRef'))`];
|
|
1366
|
+
|
|
1367
|
+
//#endregion
|
|
1368
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0006t_int32_field_kind.ts
|
|
1369
|
+
/**
|
|
1370
|
+
* 0006t_int32_field_kind — `int32` joins the field kinds the type catalog
|
|
1371
|
+
* accepts (an `integer` column).
|
|
1372
|
+
*/
|
|
1373
|
+
const TX_STATEMENTS$1 = ["ALTER TABLE substrate_type_fields DROP CONSTRAINT IF EXISTS substrate_type_fields_kind_check", `ALTER TABLE substrate_type_fields ADD CONSTRAINT substrate_type_fields_kind_check
|
|
1374
|
+
CHECK (kind IN ('text','enum','number','int32','int64','date','boolean','crossRef'))`];
|
|
1375
|
+
|
|
1376
|
+
//#endregion
|
|
1377
|
+
//#region ../../packages/substrate-dataset-pg/src/migrations/0007t_int16_field_kind.ts
|
|
1378
|
+
/**
|
|
1379
|
+
* 0007t_int16_field_kind — `int16` joins the field kinds the type catalog
|
|
1380
|
+
* accepts (a `smallint` column).
|
|
1381
|
+
*/
|
|
1382
|
+
const TX_STATEMENTS = ["ALTER TABLE substrate_type_fields DROP CONSTRAINT IF EXISTS substrate_type_fields_kind_check", `ALTER TABLE substrate_type_fields ADD CONSTRAINT substrate_type_fields_kind_check
|
|
1383
|
+
CHECK (kind IN ('text','enum','number','int16','int32','int64','date','boolean','crossRef'))`];
|
|
1272
1384
|
|
|
1273
1385
|
//#endregion
|
|
1274
1386
|
//#region ../../packages/substrate-dataset-pg/src/server-version.ts
|
|
@@ -1328,15 +1440,27 @@ async function assertServerVersion(sql) {
|
|
|
1328
1440
|
*/
|
|
1329
1441
|
const TABLES_BASELINE = {
|
|
1330
1442
|
version: "0001t_tables_baseline",
|
|
1331
|
-
txStatements: TX_STATEMENTS$
|
|
1443
|
+
txStatements: TX_STATEMENTS$6,
|
|
1332
1444
|
concurrentStatements: CONCURRENT_STATEMENTS
|
|
1333
1445
|
};
|
|
1334
1446
|
const TYPE_TABLE_VERSION = {
|
|
1335
1447
|
version: "0003t_type_table_version",
|
|
1336
|
-
txStatements: TX_STATEMENTS$
|
|
1448
|
+
txStatements: TX_STATEMENTS$4
|
|
1337
1449
|
};
|
|
1338
1450
|
const FIELD_ERROR_VERSION = {
|
|
1339
1451
|
version: "0004t_field_error_version",
|
|
1452
|
+
txStatements: TX_STATEMENTS$3
|
|
1453
|
+
};
|
|
1454
|
+
const INT64_FIELD_KIND = {
|
|
1455
|
+
version: "0005t_int64_field_kind",
|
|
1456
|
+
txStatements: TX_STATEMENTS$2
|
|
1457
|
+
};
|
|
1458
|
+
const INT32_FIELD_KIND = {
|
|
1459
|
+
version: "0006t_int32_field_kind",
|
|
1460
|
+
txStatements: TX_STATEMENTS$1
|
|
1461
|
+
};
|
|
1462
|
+
const INT16_FIELD_KIND = {
|
|
1463
|
+
version: "0007t_int16_field_kind",
|
|
1340
1464
|
txStatements: TX_STATEMENTS
|
|
1341
1465
|
};
|
|
1342
1466
|
function normalise(input) {
|
|
@@ -1358,12 +1482,12 @@ function normalise(input) {
|
|
|
1358
1482
|
*/
|
|
1359
1483
|
const UUID_CHECK_INPUT = {
|
|
1360
1484
|
version: UUID_CHECK_MIGRATION,
|
|
1361
|
-
txStatements: TX_STATEMENTS$
|
|
1485
|
+
txStatements: TX_STATEMENTS$5
|
|
1362
1486
|
};
|
|
1363
1487
|
/**
|
|
1364
1488
|
* The tables baseline, then the gated UUID CHECKs when
|
|
1365
1489
|
* `SUBSTRATE_ENFORCE_UUID_CHECK=true` (or the explicit override is true), then
|
|
1366
|
-
* the type tables' catalog version, then the field error version. Used
|
|
1490
|
+
* the type tables' catalog version, then the field error version, then the int64, int32 and int16 field kinds. Used
|
|
1367
1491
|
* by `runMigrations` and exported for unit tests.
|
|
1368
1492
|
*/
|
|
1369
1493
|
function getMigrationList(enforceUuidCheck = process.env.SUBSTRATE_ENFORCE_UUID_CHECK === "true") {
|
|
@@ -1371,7 +1495,10 @@ function getMigrationList(enforceUuidCheck = process.env.SUBSTRATE_ENFORCE_UUID_
|
|
|
1371
1495
|
TABLES_BASELINE,
|
|
1372
1496
|
...enforceUuidCheck === true ? [UUID_CHECK_INPUT] : [],
|
|
1373
1497
|
TYPE_TABLE_VERSION,
|
|
1374
|
-
FIELD_ERROR_VERSION
|
|
1498
|
+
FIELD_ERROR_VERSION,
|
|
1499
|
+
INT64_FIELD_KIND,
|
|
1500
|
+
INT32_FIELD_KIND,
|
|
1501
|
+
INT16_FIELD_KIND
|
|
1375
1502
|
].map(normalise);
|
|
1376
1503
|
}
|
|
1377
1504
|
/** @internal — the always-on list, exported for unit tests. */
|
|
@@ -1773,11 +1900,14 @@ function columnKindOf(def) {
|
|
|
1773
1900
|
* walk joins `far.id = near.f_x` (and `far.f_x = near.id`) bare, and an index
|
|
1774
1901
|
* on either side serves it; it is ordered under `"C"` where it sorts.
|
|
1775
1902
|
*/
|
|
1776
|
-
const PG_TYPE = {
|
|
1903
|
+
const PG_TYPE$1 = {
|
|
1777
1904
|
text: "text COLLATE \"C\"",
|
|
1778
1905
|
enum: "text COLLATE \"C\"",
|
|
1779
1906
|
crossRef: "text",
|
|
1780
1907
|
number: "double precision",
|
|
1908
|
+
int16: "smallint",
|
|
1909
|
+
int32: "integer",
|
|
1910
|
+
int64: "bigint",
|
|
1781
1911
|
date: "timestamptz",
|
|
1782
1912
|
boolean: "boolean"
|
|
1783
1913
|
};
|
|
@@ -1893,30 +2023,11 @@ function listenTableCache(cache) {
|
|
|
1893
2023
|
cache.listening ??= typeof pool.listen !== "function" ? Promise.resolve(false) : pool.listen("substrate_catalog", (key) => drop(key), () => drop()).then(() => true, (err) => {
|
|
1894
2024
|
console.warn(`substrate-dataset-pg: LISTEN substrate_catalog failed, type tables uncached; retrying in ${LISTEN_RETRY_MS / 1e3}s: ${err instanceof Error ? err.message : String(err)}`);
|
|
1895
2025
|
cache.listenRetryAt = Date.now() + LISTEN_RETRY_MS;
|
|
1896
|
-
dropListener(pool);
|
|
1897
2026
|
return false;
|
|
1898
2027
|
});
|
|
1899
2028
|
return cache.listening;
|
|
1900
2029
|
}
|
|
1901
2030
|
/**
|
|
1902
|
-
* Closes the pool's LISTEN connection after a refused LISTEN, so the retry
|
|
1903
|
-
* opens a fresh one: postgres.js keeps a rejected LISTEN's promise for the
|
|
1904
|
-
* channel until that connection closes, and a later `listen` only joins it.
|
|
1905
|
-
* The channel is forgotten first, so the close does not re-listen the failed
|
|
1906
|
-
* listeners; other channels are re-listened on the new connection.
|
|
1907
|
-
*
|
|
1908
|
-
* shortcut: reaches into postgres.js 3.x's `listen.sql` / `listen.channels`;
|
|
1909
|
-
* re-check on a postgres.js upgrade (the listen-retry test fails if they move).
|
|
1910
|
-
*/
|
|
1911
|
-
function dropListener(pool) {
|
|
1912
|
-
const listen = pool.listen;
|
|
1913
|
-
if (listen === void 0) return;
|
|
1914
|
-
delete listen.channels?.substrate_catalog;
|
|
1915
|
-
const conn = listen.sql;
|
|
1916
|
-
delete listen.sql;
|
|
1917
|
-
conn?.end({ timeout: 0 }).catch(() => {});
|
|
1918
|
-
}
|
|
1919
|
-
/**
|
|
1920
2031
|
* `namespace`/`type`'s descriptor through the pool's {@link TableCache}, as
|
|
1921
2032
|
* `PgDataset` reads it outside a transaction: a hit, a miss included, makes no
|
|
1922
2033
|
* query; a load that raced a notification is not cached.
|
|
@@ -2064,7 +2175,7 @@ async function ensureIn(tx, args) {
|
|
|
2064
2175
|
let changed = false;
|
|
2065
2176
|
const record = (field, kind, state, vocab) => tx`INSERT INTO substrate_type_fields (namespace, type, field, column_name, kind, state, vocab)
|
|
2066
2177
|
VALUES (${namespace}, ${type}, ${field}, ${columnName(field)}, ${kind}, ${state},
|
|
2067
|
-
${vocab === void 0 ? null :
|
|
2178
|
+
${vocab === void 0 ? null : [...vocab]})`;
|
|
2068
2179
|
const setState = (field, state) => tx`UPDATE substrate_type_fields SET state = ${state}
|
|
2069
2180
|
WHERE namespace = ${namespace} AND type = ${type} AND field = ${field}`;
|
|
2070
2181
|
const [existing] = await tx`
|
|
@@ -2096,7 +2207,7 @@ async function ensureIn(tx, args) {
|
|
|
2096
2207
|
const widened = widen(row.vocab, vocab);
|
|
2097
2208
|
if (widened !== void 0) {
|
|
2098
2209
|
if (row.state === "pending" || await ddl(tx, `ALTER TABLE ${table} ${enumCheckClauses(row.column_name, widened).join(", ")}`)) {
|
|
2099
|
-
await tx`UPDATE substrate_type_fields SET vocab = ${
|
|
2210
|
+
await tx`UPDATE substrate_type_fields SET vocab = ${[...widened]}
|
|
2100
2211
|
WHERE namespace = ${namespace} AND type = ${type} AND field = ${field}`;
|
|
2101
2212
|
if (row.state === "column") await setState(field, "promoting");
|
|
2102
2213
|
changed = true;
|
|
@@ -2163,7 +2274,7 @@ function createTableSql(table, declared, enforceUuidCheck) {
|
|
|
2163
2274
|
];
|
|
2164
2275
|
for (const [field, { kind, vocab }] of declared) {
|
|
2165
2276
|
const column = columnName(field);
|
|
2166
|
-
parts.push(`${ident(column)} ${PG_TYPE[kind]}`);
|
|
2277
|
+
parts.push(`${ident(column)} ${PG_TYPE$1[kind]}`);
|
|
2167
2278
|
if (kind === "enum") parts.push(`CONSTRAINT ${ident(enumCheckName(column))} ${enumCheck(column, vocab)}`);
|
|
2168
2279
|
}
|
|
2169
2280
|
if (enforceUuidCheck) parts.push(`CONSTRAINT id_uuid_format CHECK (id ~ '${UUID_REGEX}')`);
|
|
@@ -2273,7 +2384,7 @@ async function lockTypeTable(tx, namespace, type) {
|
|
|
2273
2384
|
}
|
|
2274
2385
|
/** The ALTER TABLE clauses giving `column` its type, and its CHECK when it is an enum. */
|
|
2275
2386
|
function addColumnClauses(column, kind, vocab) {
|
|
2276
|
-
return [`ADD COLUMN IF NOT EXISTS ${ident(column)} ${PG_TYPE[kind]}`, ...kind === "enum" ? enumCheckClauses(column, vocab) : []];
|
|
2387
|
+
return [`ADD COLUMN IF NOT EXISTS ${ident(column)} ${PG_TYPE$1[kind]}`, ...kind === "enum" ? enumCheckClauses(column, vocab) : []];
|
|
2277
2388
|
}
|
|
2278
2389
|
/**
|
|
2279
2390
|
* Runs one DDL statement in a savepoint under a short lock_timeout. False when
|
|
@@ -2656,7 +2767,7 @@ function keyExpr(field, def) {
|
|
|
2656
2767
|
const typed = typedColumn(field, def);
|
|
2657
2768
|
if (typed !== void 0) return ident(typed);
|
|
2658
2769
|
if (def?.kind === "crossRef") return indexExpressionSql("text", field, void 0, "extra");
|
|
2659
|
-
if (def?.kind === "scalar" && def.type !== "json") return indexExpressionSql(def.type === "
|
|
2770
|
+
if (def?.kind === "scalar" && def.type !== "json") return indexExpressionSql(def.type === "int16" || def.type === "int32" ? "int64" : def.type === "number" || def.type === "int64" || def.type === "date" ? def.type : "text", field, void 0, "extra");
|
|
2660
2771
|
throw new Error(`field '${field}' is not a declared scalar or crossRef field and cannot be indexed`);
|
|
2661
2772
|
}
|
|
2662
2773
|
function desiredSpecs(args) {
|
|
@@ -2908,7 +3019,7 @@ async function build(conn, args, target, spec, name, opts) {
|
|
|
2908
3019
|
}
|
|
2909
3020
|
opts.onBuild?.(spec.name);
|
|
2910
3021
|
try {
|
|
2911
|
-
await
|
|
3022
|
+
await conn.unsafe(createSql(spec, target, name, true)).signal(spec.unique ? void 0 : opts.signal);
|
|
2912
3023
|
return;
|
|
2913
3024
|
} catch (err) {
|
|
2914
3025
|
if (spec.unique) throw err;
|
|
@@ -2958,22 +3069,8 @@ async function buildEstimate(conn, target, name) {
|
|
|
2958
3069
|
0)::text AS n`;
|
|
2959
3070
|
return Number(row?.n ?? 0);
|
|
2960
3071
|
}
|
|
2961
|
-
/** Runs `text` on `conn`, cancelling it in flight when `signal` aborts. */
|
|
2962
|
-
async function cancellable$1(conn, text, signal) {
|
|
2963
|
-
const query = conn.unsafe(text);
|
|
2964
|
-
const cancel = () => {
|
|
2965
|
-
Promise.resolve(query.cancel()).catch(() => void 0);
|
|
2966
|
-
};
|
|
2967
|
-
signal?.addEventListener("abort", cancel, { once: true });
|
|
2968
|
-
try {
|
|
2969
|
-
if (signal?.aborted === true) cancel();
|
|
2970
|
-
await query;
|
|
2971
|
-
} finally {
|
|
2972
|
-
signal?.removeEventListener("abort", cancel);
|
|
2973
|
-
}
|
|
2974
|
-
}
|
|
2975
3072
|
async function dropConcurrently(conn, name, signal) {
|
|
2976
|
-
await
|
|
3073
|
+
await conn.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`).signal(signal);
|
|
2977
3074
|
}
|
|
2978
3075
|
async function mark(conn, name) {
|
|
2979
3076
|
await conn.unsafe(`COMMENT ON INDEX ${name} IS '${MARK}'`);
|
|
@@ -3212,6 +3309,10 @@ function buildMetadataColumnPredicate(sql, field, predicate, alias) {
|
|
|
3212
3309
|
case "endsWith":
|
|
3213
3310
|
if (typeof operand === "string") frag = sql`${col} ILIKE ${`%${escapeLike(operand)}`}`;
|
|
3214
3311
|
break;
|
|
3312
|
+
case "like":
|
|
3313
|
+
case "notLike":
|
|
3314
|
+
frag = likeFragment(sql, col, field, op, operand);
|
|
3315
|
+
break;
|
|
3215
3316
|
case "gt":
|
|
3216
3317
|
frag = sql`${col} COLLATE "C" > ${metaStr(field, op, operand)}`;
|
|
3217
3318
|
break;
|
|
@@ -3298,7 +3399,7 @@ function memberOf(sql, col, values, operand) {
|
|
|
3298
3399
|
* `sortExpr` is the single source for ORDER BY and both halves of the keyset
|
|
3299
3400
|
* cursor comparison, so sorting and paging cannot diverge.
|
|
3300
3401
|
*/
|
|
3301
|
-
const
|
|
3402
|
+
const noComputed = () => void 0;
|
|
3302
3403
|
/**
|
|
3303
3404
|
* Maps a FieldDef to the pg cast applied to its `body ->> field` text
|
|
3304
3405
|
* extraction. Returns `null` for kinds where text compare is correct (string),
|
|
@@ -3306,6 +3407,9 @@ const noDerived = () => void 0;
|
|
|
3306
3407
|
*
|
|
3307
3408
|
* Type → cast table:
|
|
3308
3409
|
* field.number() / 'number' → 'numeric'
|
|
3410
|
+
* field.int16() / 'int16' → 'bigint'
|
|
3411
|
+
* field.int32() / 'int32' → 'bigint'
|
|
3412
|
+
* field.int64() / 'int64' → 'bigint'
|
|
3309
3413
|
* field.date() / 'date' → 'timestamptz'
|
|
3310
3414
|
* field.boolean() / 'boolean' → 'boolean'
|
|
3311
3415
|
* field.string() / 'string' → null (text compare is correct)
|
|
@@ -3320,6 +3424,9 @@ function resolveBodyOrderByCast(def) {
|
|
|
3320
3424
|
if (def.kind !== "scalar") return null;
|
|
3321
3425
|
switch (def.type) {
|
|
3322
3426
|
case "number": return "numeric";
|
|
3427
|
+
case "int16":
|
|
3428
|
+
case "int32":
|
|
3429
|
+
case "int64": return "bigint";
|
|
3323
3430
|
case "date": return "timestamptz";
|
|
3324
3431
|
case "boolean": return "boolean";
|
|
3325
3432
|
default: return null;
|
|
@@ -3330,10 +3437,10 @@ function resolveBodyOrderByCast(def) {
|
|
|
3330
3437
|
* SELECT list's column of that name first, and a date column is projected
|
|
3331
3438
|
* under its own name as text, which no index on the column serves.
|
|
3332
3439
|
*/
|
|
3333
|
-
function buildOrderBy(sql, orderBy, castFor, table,
|
|
3440
|
+
function buildOrderBy(sql, orderBy, castFor, table, computedFor = noComputed, alias) {
|
|
3334
3441
|
if (orderBy.length === 0) return sql``;
|
|
3335
3442
|
const parts = [];
|
|
3336
|
-
for (const [field, dir] of orderBy) parts.push(orderByFragment(sql, field, dir, castFor(field), table,
|
|
3443
|
+
for (const [field, dir] of orderBy) parts.push(orderByFragment(sql, field, dir, castFor(field), table, computedFor, alias));
|
|
3337
3444
|
if (parts.length === 1) return sql`ORDER BY ${parts[0]}`;
|
|
3338
3445
|
const commaFragment = sql`,`;
|
|
3339
3446
|
const interleaved = [parts[0]];
|
|
@@ -3347,10 +3454,10 @@ function buildOrderBy(sql, orderBy, castFor, table, derivedFor = noDerived, alia
|
|
|
3347
3454
|
* Do not case out field kinds anywhere else: if sorting and paging use
|
|
3348
3455
|
* different expressions, rows silently vanish from later pages.
|
|
3349
3456
|
*/
|
|
3350
|
-
function sortExpr(sql, field, bodyCast, table,
|
|
3457
|
+
function sortExpr(sql, field, bodyCast, table, computedFor, alias) {
|
|
3351
3458
|
if (field === "id") return sql`id`;
|
|
3352
|
-
const
|
|
3353
|
-
if (
|
|
3459
|
+
const computed = computedFor(field);
|
|
3460
|
+
if (computed !== void 0) return computed;
|
|
3354
3461
|
if (field === "shortId") return sql`s.short_id`;
|
|
3355
3462
|
if (isMetadataField(field)) {
|
|
3356
3463
|
const col = METADATA_COLUMNS$1[field];
|
|
@@ -3369,7 +3476,7 @@ function sortExpr(sql, field, bodyCast, table, derivedFor, alias) {
|
|
|
3369
3476
|
* Aliases are POSITIONAL (`__sort_0`, ...), not field-derived: pg truncates
|
|
3370
3477
|
* identifiers at 63 bytes, so long field names could collide.
|
|
3371
3478
|
*/
|
|
3372
|
-
function buildSortKeyProjection(sql, orderBy, castFor, table,
|
|
3479
|
+
function buildSortKeyProjection(sql, orderBy, castFor, table, computedFor = noComputed) {
|
|
3373
3480
|
const aliases = /* @__PURE__ */ new Map();
|
|
3374
3481
|
const fragments = [];
|
|
3375
3482
|
for (const [field] of orderBy) {
|
|
@@ -3377,8 +3484,8 @@ function buildSortKeyProjection(sql, orderBy, castFor, table, derivedFor = noDer
|
|
|
3377
3484
|
const alias = `__sort_${aliases.size}`;
|
|
3378
3485
|
aliases.set(field, alias);
|
|
3379
3486
|
const cast = castFor(field);
|
|
3380
|
-
const expr = sortExpr(sql, field, cast, table,
|
|
3381
|
-
fragments.push(sortKeyIsTimestamp(field, cast, table) ? sql`, (${expr})::text AS ${sql(alias)}` : sql`, ${expr} AS ${sql(alias)}`);
|
|
3487
|
+
const expr = sortExpr(sql, field, cast, table, computedFor);
|
|
3488
|
+
fragments.push(sortKeyIsTimestamp(field, cast, table, computedFor) ? sql`, (${expr})::text AS ${sql(alias)}` : sql`, ${expr} AS ${sql(alias)}`);
|
|
3382
3489
|
}
|
|
3383
3490
|
if (fragments.length === 0) return {
|
|
3384
3491
|
fragment: sql``,
|
|
@@ -3390,7 +3497,8 @@ function buildSortKeyProjection(sql, orderBy, castFor, table, derivedFor = noDer
|
|
|
3390
3497
|
};
|
|
3391
3498
|
}
|
|
3392
3499
|
/** Whether `buildSortKeyProjection` projects the field's key as timestamptz text. */
|
|
3393
|
-
function sortKeyIsTimestamp(field, cast, table) {
|
|
3500
|
+
function sortKeyIsTimestamp(field, cast, table, computedFor = noComputed) {
|
|
3501
|
+
if (computedFor(field) !== void 0) return computedFor.typeOf?.(field) === "date";
|
|
3394
3502
|
if (field === "createdAt" || field === "updatedAt") return true;
|
|
3395
3503
|
if (isMetadataField(field)) return false;
|
|
3396
3504
|
const column = table.column(field);
|
|
@@ -3402,7 +3510,7 @@ function sortKeyIsTimestamp(field, cast, table) {
|
|
|
3402
3510
|
* directions and NULLS LAST; a timestamp key, projected as text, casts back
|
|
3403
3511
|
* so it orders as a timestamp.
|
|
3404
3512
|
*/
|
|
3405
|
-
function buildProjectedOrderBy(sql, orderBy, aliases, castFor, table, rel) {
|
|
3513
|
+
function buildProjectedOrderBy(sql, orderBy, aliases, castFor, table, rel, computedFor = noComputed) {
|
|
3406
3514
|
const parts = [];
|
|
3407
3515
|
for (const [field, dir] of orderBy) {
|
|
3408
3516
|
let expr;
|
|
@@ -3410,7 +3518,7 @@ function buildProjectedOrderBy(sql, orderBy, aliases, castFor, table, rel) {
|
|
|
3410
3518
|
else {
|
|
3411
3519
|
const alias = aliases.get(field);
|
|
3412
3520
|
if (alias === void 0) throw new Error(`orderBy key '${field}' has no projected alias`);
|
|
3413
|
-
expr = sortKeyIsTimestamp(field, castFor(field), table) ? `${rel}.${alias}::timestamptz` : `${rel}.${alias}`;
|
|
3521
|
+
expr = sortKeyIsTimestamp(field, castFor(field), table, computedFor) ? `${rel}.${alias}::timestamptz` : `${rel}.${alias}`;
|
|
3414
3522
|
}
|
|
3415
3523
|
parts.push(`${expr} ${dir === "asc" ? "ASC" : "DESC"} NULLS LAST`);
|
|
3416
3524
|
}
|
|
@@ -3420,9 +3528,14 @@ function buildProjectedOrderBy(sql, orderBy, aliases, castFor, table, rel) {
|
|
|
3420
3528
|
* The parameter side of a comparison against `sortExpr(field)`, cast to match
|
|
3421
3529
|
* it. Keep its cases in step with `sortExpr`.
|
|
3422
3530
|
*/
|
|
3423
|
-
function sortParam(sql, field, value, bodyCast, table,
|
|
3531
|
+
function sortParam(sql, field, value, bodyCast, table, computedFor) {
|
|
3424
3532
|
if (field === "id") return sql`${idToText(value)}`;
|
|
3425
|
-
if (
|
|
3533
|
+
if (computedFor(field) !== void 0) {
|
|
3534
|
+
const type = computedFor.typeOf?.(field) ?? "number";
|
|
3535
|
+
if (type === "int64") return sql`${String(value)}::bigint`;
|
|
3536
|
+
if (type === "date") return tsParam(sql, value);
|
|
3537
|
+
if (type === "string") return sql`${String(value)}::text`;
|
|
3538
|
+
if (type === "boolean") return sql`${Boolean(value)}::boolean`;
|
|
3426
3539
|
if (typeof value !== "number" || !Number.isFinite(value)) throw new CursorDecodeError(`cursor value for '${field}' is not a finite number`);
|
|
3427
3540
|
return sql`${value}::float8`;
|
|
3428
3541
|
}
|
|
@@ -3433,13 +3546,14 @@ function sortParam(sql, field, value, bodyCast, table, derivedFor) {
|
|
|
3433
3546
|
if (column !== void 0) switch (columnAs(column.kind)) {
|
|
3434
3547
|
case "date": return tsParam(sql, value);
|
|
3435
3548
|
case "number": return sql`${value}::numeric`;
|
|
3549
|
+
case "int64": return sql`${String(value)}::bigint`;
|
|
3436
3550
|
case "boolean": return sql`${Boolean(value)}::boolean`;
|
|
3437
3551
|
default: return sql`${String(value)}::text`;
|
|
3438
3552
|
}
|
|
3439
3553
|
return castParam(sql, value, bodyCast);
|
|
3440
3554
|
}
|
|
3441
|
-
function orderByFragment(sql, field, dir, bodyCast, table,
|
|
3442
|
-
const expr = sortExpr(sql, field, bodyCast, table,
|
|
3555
|
+
function orderByFragment(sql, field, dir, bodyCast, table, computedFor, alias) {
|
|
3556
|
+
const expr = sortExpr(sql, field, bodyCast, table, computedFor, alias);
|
|
3443
3557
|
return dir === "asc" ? sql`${expr} ASC NULLS LAST` : sql`${expr} DESC NULLS LAST`;
|
|
3444
3558
|
}
|
|
3445
3559
|
/**
|
|
@@ -3451,6 +3565,7 @@ function bodyOrderByExpr(sql, field, cast, table) {
|
|
|
3451
3565
|
switch (cast) {
|
|
3452
3566
|
case null: return bodyText(sql, [field], void 0, table.jsonb);
|
|
3453
3567
|
case "numeric": return bodyNum(sql, [field], void 0, table.jsonb);
|
|
3568
|
+
case "bigint": return bodyInt64(sql, [field], void 0, table.jsonb);
|
|
3454
3569
|
case "timestamptz": return bodyTs(sql, [field], void 0, table.jsonb);
|
|
3455
3570
|
case "boolean": return sql`${bodyRawText(sql, [field], void 0, table.jsonb)}::boolean`;
|
|
3456
3571
|
}
|
|
@@ -3474,16 +3589,16 @@ function bodyOrderByExpr(sql, field, cast, table) {
|
|
|
3474
3589
|
* We build the disjunction explicitly — postgres's ROW(...) tuple compare
|
|
3475
3590
|
* would be simpler but doesn't honour per-column direction mixing.
|
|
3476
3591
|
*/
|
|
3477
|
-
function buildCursorCompare(sql, orderBy, values, cursorId, castFor, table,
|
|
3592
|
+
function buildCursorCompare(sql, orderBy, values, cursorId, castFor, table, computedFor = noComputed) {
|
|
3478
3593
|
const resolved = resolveCursorKeys(orderBy, values, cursorId);
|
|
3479
3594
|
const [lead, ...rest] = resolved;
|
|
3480
3595
|
if (lead === void 0) return null;
|
|
3481
|
-
if (lead.field === "id") return cursorDisjunction(sql, resolved, castFor, table,
|
|
3482
|
-
const restCmp = cursorDisjunction(sql, rest, castFor, table,
|
|
3596
|
+
if (lead.field === "id") return cursorDisjunction(sql, resolved, castFor, table, computedFor);
|
|
3597
|
+
const restCmp = cursorDisjunction(sql, rest, castFor, table, computedFor);
|
|
3483
3598
|
const cast = castFor(lead.field);
|
|
3484
|
-
const eq = cursorEqFragment(sql, lead.field, lead.value, cast, table,
|
|
3599
|
+
const eq = cursorEqFragment(sql, lead.field, lead.value, cast, table, computedFor);
|
|
3485
3600
|
if (lead.value === null || lead.value === void 0) return combineAnd(sql, [eq, restCmp]);
|
|
3486
|
-
const cmp = (op) => sql`${sortExpr(sql, lead.field, cast, table,
|
|
3601
|
+
const cmp = (op) => sql`${sortExpr(sql, lead.field, cast, table, computedFor)} ${sql.unsafe(op)} ${sortParam(sql, lead.field, lead.value, cast, table, computedFor)}`;
|
|
3487
3602
|
const asc = lead.dir === "asc";
|
|
3488
3603
|
return combineAnd(sql, [cmp(asc ? ">=" : "<="), combineOr(sql, [cmp(asc ? ">" : "<"), combineAnd(sql, [eq, restCmp])])]);
|
|
3489
3604
|
}
|
|
@@ -3494,11 +3609,11 @@ function buildCursorCompare(sql, orderBy, values, cursorId, castFor, table, deri
|
|
|
3494
3609
|
* tail query needs no other cursor condition. `createdAt` and `updatedAt` count
|
|
3495
3610
|
* as nullable: their columns are nullable in both baselines.
|
|
3496
3611
|
*/
|
|
3497
|
-
function buildCursorNullTail(sql, orderBy, values, castFor, table,
|
|
3612
|
+
function buildCursorNullTail(sql, orderBy, values, castFor, table, computedFor = noComputed) {
|
|
3498
3613
|
const lead = orderBy[0];
|
|
3499
3614
|
if (lead === void 0 || lead[0] === "id") return null;
|
|
3500
3615
|
if (values[0] === null || values[0] === void 0) return null;
|
|
3501
|
-
return sql`${sortExpr(sql, lead[0], castFor(lead[0]), table,
|
|
3616
|
+
return sql`${sortExpr(sql, lead[0], castFor(lead[0]), table, computedFor)} IS NULL`;
|
|
3502
3617
|
}
|
|
3503
3618
|
/** Pairs each orderBy key with its cursor value: `id` from `cursorId`, the rest from `values` in order. */
|
|
3504
3619
|
function resolveCursorKeys(orderBy, values, cursorId) {
|
|
@@ -3520,16 +3635,16 @@ function resolveCursorKeys(orderBy, values, cursorId) {
|
|
|
3520
3635
|
return resolved;
|
|
3521
3636
|
}
|
|
3522
3637
|
/** The OR over i of (eq₁ AND ... AND eqᵢ₋₁ AND cmpᵢ). */
|
|
3523
|
-
function cursorDisjunction(sql, resolved, castFor, table,
|
|
3638
|
+
function cursorDisjunction(sql, resolved, castFor, table, computedFor) {
|
|
3524
3639
|
const orParts = [];
|
|
3525
3640
|
for (let i = 0; i < resolved.length; i++) {
|
|
3526
3641
|
const parts = [];
|
|
3527
3642
|
for (let j = 0; j < i; j++) {
|
|
3528
3643
|
const prior = resolved[j];
|
|
3529
|
-
parts.push(cursorEqFragment(sql, prior.field, prior.value, castFor(prior.field), table,
|
|
3644
|
+
parts.push(cursorEqFragment(sql, prior.field, prior.value, castFor(prior.field), table, computedFor));
|
|
3530
3645
|
}
|
|
3531
3646
|
const cur = resolved[i];
|
|
3532
|
-
parts.push(cursorCmpFragment(sql, cur.field, cur.dir, cur.value, castFor(cur.field), table,
|
|
3647
|
+
parts.push(cursorCmpFragment(sql, cur.field, cur.dir, cur.value, castFor(cur.field), table, computedFor));
|
|
3533
3648
|
orParts.push(combineAnd(sql, parts));
|
|
3534
3649
|
}
|
|
3535
3650
|
return combineOr(sql, orParts);
|
|
@@ -3538,10 +3653,10 @@ function cursorDisjunction(sql, resolved, castFor, table, derivedFor) {
|
|
|
3538
3653
|
* Equality against the sort key, for the `eq₁ AND ... AND cmpᵢ` prefix of the
|
|
3539
3654
|
* keyset disjunction. A NULL sort key compares with `IS NULL`, never `= NULL`.
|
|
3540
3655
|
*/
|
|
3541
|
-
function cursorEqFragment(sql, field, value, bodyCast, table,
|
|
3542
|
-
const expr = sortExpr(sql, field, bodyCast, table,
|
|
3656
|
+
function cursorEqFragment(sql, field, value, bodyCast, table, computedFor) {
|
|
3657
|
+
const expr = sortExpr(sql, field, bodyCast, table, computedFor);
|
|
3543
3658
|
if (value === null || value === void 0) return sql`${expr} IS NULL`;
|
|
3544
|
-
return sql`${expr} = ${sortParam(sql, field, value, bodyCast, table,
|
|
3659
|
+
return sql`${expr} = ${sortParam(sql, field, value, bodyCast, table, computedFor)}`;
|
|
3545
3660
|
}
|
|
3546
3661
|
/**
|
|
3547
3662
|
* "Strictly after the cursor" for one sort key, under `NULLS LAST`. On a
|
|
@@ -3557,10 +3672,10 @@ function cursorEqFragment(sql, field, value, bodyCast, table, derivedFor) {
|
|
|
3557
3672
|
* with `eq` on the preceding keys, and returning `IS NULL` here would
|
|
3558
3673
|
* re-emit every null row that has already been served.
|
|
3559
3674
|
*/
|
|
3560
|
-
function cursorCmpFragment(sql, field, dir, value, bodyCast, table,
|
|
3561
|
-
const expr = sortExpr(sql, field, bodyCast, table,
|
|
3675
|
+
function cursorCmpFragment(sql, field, dir, value, bodyCast, table, computedFor) {
|
|
3676
|
+
const expr = sortExpr(sql, field, bodyCast, table, computedFor);
|
|
3562
3677
|
if (value === null || value === void 0) return sql`false`;
|
|
3563
|
-
const param = sortParam(sql, field, value, bodyCast, table,
|
|
3678
|
+
const param = sortParam(sql, field, value, bodyCast, table, computedFor);
|
|
3564
3679
|
if (field === "id") return dir === "asc" ? sql`${expr} > ${param}` : sql`${expr} < ${param}`;
|
|
3565
3680
|
return dir === "asc" ? sql`(${expr} > ${param} OR ${expr} IS NULL)` : sql`(${expr} < ${param} OR ${expr} IS NULL)`;
|
|
3566
3681
|
}
|
|
@@ -3576,6 +3691,264 @@ function castParam(sql, value, cast) {
|
|
|
3576
3691
|
return sql`${param}${sql.unsafe(`::${cast}`)}`;
|
|
3577
3692
|
}
|
|
3578
3693
|
|
|
3694
|
+
//#endregion
|
|
3695
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/expression.ts
|
|
3696
|
+
/**
|
|
3697
|
+
* Compiles a read-grammar `Expr` to a Postgres scalar expression that returns
|
|
3698
|
+
* what the memory evaluator (`evalExpr`) returns. Every literal is a bound
|
|
3699
|
+
* parameter; the only inlined constants are date units drawn from
|
|
3700
|
+
* `DATE_UNITS`. Types: number is float8, int64 bigint, string text, boolean
|
|
3701
|
+
* boolean, date timestamptz.
|
|
3702
|
+
*/
|
|
3703
|
+
const KIND_TYPE = {
|
|
3704
|
+
text: "string",
|
|
3705
|
+
enum: "string",
|
|
3706
|
+
crossRef: "string",
|
|
3707
|
+
number: "number",
|
|
3708
|
+
int16: "int64",
|
|
3709
|
+
int32: "int64",
|
|
3710
|
+
int64: "int64",
|
|
3711
|
+
date: "date",
|
|
3712
|
+
boolean: "boolean"
|
|
3713
|
+
};
|
|
3714
|
+
const AS = {
|
|
3715
|
+
string: "text",
|
|
3716
|
+
number: "number",
|
|
3717
|
+
int64: "int64",
|
|
3718
|
+
date: "date",
|
|
3719
|
+
boolean: "boolean"
|
|
3720
|
+
};
|
|
3721
|
+
const PG_TYPE = {
|
|
3722
|
+
string: "text",
|
|
3723
|
+
number: "float8",
|
|
3724
|
+
int64: "bigint",
|
|
3725
|
+
date: "timestamptz",
|
|
3726
|
+
boolean: "boolean"
|
|
3727
|
+
};
|
|
3728
|
+
/** The compiled expression and its type (`null` for a bare null literal). */
|
|
3729
|
+
function compileExprTyped(sql, expr, ctx) {
|
|
3730
|
+
const u = (s) => sql.unsafe(s);
|
|
3731
|
+
const cast = (t) => PG_TYPE[t];
|
|
3732
|
+
/** `NULL` typed as `t`, for a slot that needs a type. */
|
|
3733
|
+
const nul = (t) => u(`NULL::${cast(t)}`);
|
|
3734
|
+
/** `n` as type `t`: a null literal takes the type, int64 widens to float8. */
|
|
3735
|
+
const as = (n, t) => n.t === null ? nul(t) : n.t === "int64" && t === "number" ? sql`(${n.f})::float8` : n.f;
|
|
3736
|
+
const join = (fs, op) => {
|
|
3737
|
+
const out = [fs[0]];
|
|
3738
|
+
for (const f of fs.slice(1)) out.push(u(op), f);
|
|
3739
|
+
return out;
|
|
3740
|
+
};
|
|
3741
|
+
const interleave = (fs, op) => sql`(${join(fs, op)})`;
|
|
3742
|
+
const isInt = (a, b) => a.t === "int64" && (b.t === "int64" || b.bigint !== void 0) || b.t === "int64" && a.bigint !== void 0;
|
|
3743
|
+
const big = (n) => n.t === "int64" ? n.f : n.bigint;
|
|
3744
|
+
const toDate = (n) => n.t === "date" ? n.f : n.t === null ? nul("date") : sql`substrate_ts(to_jsonb((${n.f})::text))`;
|
|
3745
|
+
/** The type branches unify to, and each branch cast to it (an int64 beside a number widens). */
|
|
3746
|
+
const unifyAll = (ns) => {
|
|
3747
|
+
const types = ns.map((n) => n.t).filter((t) => t !== null);
|
|
3748
|
+
const numeric_ = types.length > 0 && types.every((t) => t === "number" || t === "int64");
|
|
3749
|
+
const int = types.includes("int64") && ns.every((n) => n.t === "int64" || n.t === null || n.bigint);
|
|
3750
|
+
const t = !numeric_ ? types[0] ?? null : int ? "int64" : "number";
|
|
3751
|
+
return {
|
|
3752
|
+
t,
|
|
3753
|
+
fs: ns.map((n) => t === null ? n.f : t === "int64" && n.t !== "int64" && n.bigint ? n.bigint : as(n, t))
|
|
3754
|
+
};
|
|
3755
|
+
};
|
|
3756
|
+
const go = (e) => {
|
|
3757
|
+
if (e === null) return {
|
|
3758
|
+
f: u("NULL"),
|
|
3759
|
+
t: null
|
|
3760
|
+
};
|
|
3761
|
+
if (typeof e === "boolean") return {
|
|
3762
|
+
f: sql`${e}::boolean`,
|
|
3763
|
+
t: "boolean"
|
|
3764
|
+
};
|
|
3765
|
+
if (typeof e === "number") return {
|
|
3766
|
+
f: sql`${String(e)}::float8`,
|
|
3767
|
+
t: "number",
|
|
3768
|
+
...Number.isSafeInteger(e) ? { bigint: sql`${String(e)}::bigint` } : {}
|
|
3769
|
+
};
|
|
3770
|
+
if (typeof e === "string") return field(e);
|
|
3771
|
+
if ("lit" in e) return {
|
|
3772
|
+
f: sql`${e.lit}::text`,
|
|
3773
|
+
t: "string"
|
|
3774
|
+
};
|
|
3775
|
+
if ("int64" in e) return {
|
|
3776
|
+
f: sql`${e.int64}::bigint`,
|
|
3777
|
+
t: "int64"
|
|
3778
|
+
};
|
|
3779
|
+
const key = Object.keys(e)[0];
|
|
3780
|
+
return call(key, e[key]);
|
|
3781
|
+
};
|
|
3782
|
+
const field = (name) => {
|
|
3783
|
+
const out = ctx.outputs?.get(name);
|
|
3784
|
+
const col = ctx.table.column(name);
|
|
3785
|
+
const meta = out === void 0 ? metadataExprType(name) : void 0;
|
|
3786
|
+
if (meta !== void 0) {
|
|
3787
|
+
const c = u(metadataColumnSql(name, ctx.alias));
|
|
3788
|
+
return meta === "number" ? {
|
|
3789
|
+
f: sql`(${c})::float8`,
|
|
3790
|
+
t: meta
|
|
3791
|
+
} : {
|
|
3792
|
+
f: c,
|
|
3793
|
+
t: meta
|
|
3794
|
+
};
|
|
3795
|
+
}
|
|
3796
|
+
const t = ctx.fieldType?.(name) ?? (col ? KIND_TYPE[col.kind] : void 0);
|
|
3797
|
+
if (t === void 0) throw new Error(`internal: no type for expression field '${name}'`);
|
|
3798
|
+
if (out !== void 0) return {
|
|
3799
|
+
f: t === "number" ? sql`(${out})::float8` : out,
|
|
3800
|
+
t
|
|
3801
|
+
};
|
|
3802
|
+
const f = fieldExprSql(ctx.table, name, AS[t], ctx.alias);
|
|
3803
|
+
const wide = t === "number" ? `(${f})::float8` : col?.kind === "int16" || col?.kind === "int32" ? `(${f})::bigint` : f;
|
|
3804
|
+
return {
|
|
3805
|
+
f: u(wide),
|
|
3806
|
+
t
|
|
3807
|
+
};
|
|
3808
|
+
};
|
|
3809
|
+
const call = (key, args) => {
|
|
3810
|
+
const n = key === "case" ? [] : args.map(go);
|
|
3811
|
+
const [a, b, c] = n;
|
|
3812
|
+
const bool = (f) => ({
|
|
3813
|
+
f,
|
|
3814
|
+
t: "boolean"
|
|
3815
|
+
});
|
|
3816
|
+
const str = (f) => ({
|
|
3817
|
+
f,
|
|
3818
|
+
t: "string"
|
|
3819
|
+
});
|
|
3820
|
+
const text = (x) => x.t === null ? nul("string") : x.f;
|
|
3821
|
+
const compare = (x, y, op) => {
|
|
3822
|
+
if (x.t === "string" || y.t === "string") return sql`((${as(x, "string")}) COLLATE "C" ${u(op)} (${as(y, "string")}) COLLATE "C")`;
|
|
3823
|
+
if (isInt(x, y)) return sql`(${big(x)} ${u(op)} ${big(y)})`;
|
|
3824
|
+
return sql`(${x.f} ${u(op)} ${y.f})`;
|
|
3825
|
+
};
|
|
3826
|
+
const cmp = (op) => bool(compare(a, b, op));
|
|
3827
|
+
switch (key) {
|
|
3828
|
+
case "add":
|
|
3829
|
+
case "sub":
|
|
3830
|
+
case "mul": {
|
|
3831
|
+
const op = {
|
|
3832
|
+
add: "+",
|
|
3833
|
+
sub: "-",
|
|
3834
|
+
mul: "*"
|
|
3835
|
+
}[key];
|
|
3836
|
+
const t = isInt(a, b) ? "int64" : "number";
|
|
3837
|
+
if (a.t === null || b.t === null) return {
|
|
3838
|
+
f: nul(t),
|
|
3839
|
+
t
|
|
3840
|
+
};
|
|
3841
|
+
return t === "int64" ? {
|
|
3842
|
+
f: sql`(${big(a)} ${u(op)} ${big(b)})`,
|
|
3843
|
+
t
|
|
3844
|
+
} : {
|
|
3845
|
+
f: sql`(${as(a, "number")} ${u(op)} ${as(b, "number")})`,
|
|
3846
|
+
t
|
|
3847
|
+
};
|
|
3848
|
+
}
|
|
3849
|
+
case "mod":
|
|
3850
|
+
if (a.t === null || b.t === null) return {
|
|
3851
|
+
f: nul("int64"),
|
|
3852
|
+
t: "int64"
|
|
3853
|
+
};
|
|
3854
|
+
return {
|
|
3855
|
+
f: sql`(${big(a)} % NULLIF(${big(b)}, 0))`,
|
|
3856
|
+
t: "int64"
|
|
3857
|
+
};
|
|
3858
|
+
case "div":
|
|
3859
|
+
if (a.t === null || b.t === null) return {
|
|
3860
|
+
f: nul("number"),
|
|
3861
|
+
t: "number"
|
|
3862
|
+
};
|
|
3863
|
+
return {
|
|
3864
|
+
f: sql`(${as(a, "number")} / NULLIF(${as(b, "number")}, 0))`,
|
|
3865
|
+
t: "number"
|
|
3866
|
+
};
|
|
3867
|
+
case "neg":
|
|
3868
|
+
if (a.t === null) return {
|
|
3869
|
+
f: nul("number"),
|
|
3870
|
+
t: "number"
|
|
3871
|
+
};
|
|
3872
|
+
return {
|
|
3873
|
+
f: sql`(-(${a.f}))`,
|
|
3874
|
+
t: a.t === "int64" ? "int64" : "number"
|
|
3875
|
+
};
|
|
3876
|
+
case "eq": return cmp("=");
|
|
3877
|
+
case "neq": return cmp("<>");
|
|
3878
|
+
case "gt": return cmp(">");
|
|
3879
|
+
case "gte": return cmp(">=");
|
|
3880
|
+
case "lt": return cmp("<");
|
|
3881
|
+
case "lte": return cmp("<=");
|
|
3882
|
+
case "and":
|
|
3883
|
+
case "or": return bool(interleave(n.map((x) => as(x, "boolean")), key === "and" ? " AND " : " OR "));
|
|
3884
|
+
case "not": return bool(sql`(NOT ${as(a, "boolean")})`);
|
|
3885
|
+
case "isNull": return bool(sql`((${a.f}) IS NULL)`);
|
|
3886
|
+
case "in": {
|
|
3887
|
+
const rest = n.slice(1).map((r) => a.t === "int64" && r.bigint ? r.bigint : r.f);
|
|
3888
|
+
return bool(sql`(${a.f} IN (${join(rest, ",")}))`);
|
|
3889
|
+
}
|
|
3890
|
+
case "length": return {
|
|
3891
|
+
f: sql`char_length(${text(a)})::float8`,
|
|
3892
|
+
t: "number"
|
|
3893
|
+
};
|
|
3894
|
+
case "lower": return str(sql`lower((${text(a)}) COLLATE "C")`);
|
|
3895
|
+
case "upper": return str(sql`upper((${text(a)}) COLLATE "C")`);
|
|
3896
|
+
case "substring": {
|
|
3897
|
+
const int = (x, lo) => sql`LEAST(GREATEST(floor(${as(x, "number")} + 0.5), ${u(String(lo))}), 2147483647)::int`;
|
|
3898
|
+
return str(sql`substring(${text(a)} from ${int(b, -2147483648)} for GREATEST(${int(c, 0)}, 0))`);
|
|
3899
|
+
}
|
|
3900
|
+
case "concat": return str(interleave(n.map(text), " || "));
|
|
3901
|
+
case "like":
|
|
3902
|
+
case "notLike": return bool(sql`(${u(key === "like" ? "" : "NOT ")}(${text(a)} LIKE ${text(b)} ESCAPE '\\'))`);
|
|
3903
|
+
case "regexpMatch": return bool(sql`((${text(a)}) COLLATE "default" ~ ${text(b)})`);
|
|
3904
|
+
case "regexpReplace": return str(sql`regexp_replace(${text(a)}, ${text(b)}, ${text(c)})`);
|
|
3905
|
+
case "dateTrunc":
|
|
3906
|
+
case "extract": {
|
|
3907
|
+
const unit = args[0].lit;
|
|
3908
|
+
if (!DATE_UNITS.includes(unit)) throw new Error(`internal: '${unit}' is not a date unit`);
|
|
3909
|
+
const d = sql`(${toDate(b)}) AT TIME ZONE 'UTC'`;
|
|
3910
|
+
return key === "extract" ? {
|
|
3911
|
+
f: sql`extract(${u(unit)} FROM ${d})::float8`,
|
|
3912
|
+
t: "number"
|
|
3913
|
+
} : {
|
|
3914
|
+
f: sql`(date_trunc(${u(`'${unit}'`)}, ${d}) AT TIME ZONE 'UTC')`,
|
|
3915
|
+
t: "date"
|
|
3916
|
+
};
|
|
3917
|
+
}
|
|
3918
|
+
case "case": {
|
|
3919
|
+
const branches = args.slice(0, -1).map((p) => p.map(go));
|
|
3920
|
+
const els = go(args[args.length - 1]);
|
|
3921
|
+
const { t, fs } = unifyAll([...branches.map(([, th]) => th), els]);
|
|
3922
|
+
const parts = [u("CASE")];
|
|
3923
|
+
branches.forEach(([cond], i) => {
|
|
3924
|
+
parts.push(u("WHEN"), as(cond, "boolean"), u("THEN"), fs[i]);
|
|
3925
|
+
});
|
|
3926
|
+
parts.push(u("ELSE"), fs[fs.length - 1], u("END"));
|
|
3927
|
+
return {
|
|
3928
|
+
f: sql`(${parts})`,
|
|
3929
|
+
t
|
|
3930
|
+
};
|
|
3931
|
+
}
|
|
3932
|
+
case "coalesce": {
|
|
3933
|
+
const { t, fs } = unifyAll(n);
|
|
3934
|
+
return {
|
|
3935
|
+
f: sql`COALESCE(${join(fs, ",")})`,
|
|
3936
|
+
t
|
|
3937
|
+
};
|
|
3938
|
+
}
|
|
3939
|
+
default: throw new Error(`unknown function '${key}'`);
|
|
3940
|
+
}
|
|
3941
|
+
};
|
|
3942
|
+
const root = go(expr);
|
|
3943
|
+
return {
|
|
3944
|
+
fragment: root.f,
|
|
3945
|
+
type: root.t
|
|
3946
|
+
};
|
|
3947
|
+
}
|
|
3948
|
+
function compileExpr(sql, expr, ctx) {
|
|
3949
|
+
return compileExprTyped(sql, expr, ctx).fragment;
|
|
3950
|
+
}
|
|
3951
|
+
|
|
3579
3952
|
//#endregion
|
|
3580
3953
|
//#region ../../packages/substrate-dataset-pg/src/pg-sql/predicates-json-path.ts
|
|
3581
3954
|
/**
|
|
@@ -3699,8 +4072,9 @@ function buildJsonPathPredicate(sql, field, predicate, alias, jsonb) {
|
|
|
3699
4072
|
* is-null filter, and a compare on an absent value is FALSE, never NULL, so a
|
|
3700
4073
|
* caller's NOT stays sound. Throws `UnsupportedFilterError` as the body path does.
|
|
3701
4074
|
*/
|
|
3702
|
-
function buildColumnPredicate(sql, field, table, column,
|
|
4075
|
+
function buildColumnPredicate(sql, field, table, column, rawPredicate, alias) {
|
|
3703
4076
|
const as = columnAs(column.kind);
|
|
4077
|
+
const predicate = as === "int64" ? int64Operands(field, rawPredicate) : rawPredicate;
|
|
3704
4078
|
const read = fieldExpr(sql, table, field, as, alias);
|
|
3705
4079
|
/**
|
|
3706
4080
|
* A promoting enum field reads its `$raw` string over the rest, as `decodeRow`
|
|
@@ -3718,6 +4092,7 @@ function buildColumnPredicate(sql, field, table, column, predicate, alias) {
|
|
|
3718
4092
|
if (typeof v === "string") return sql`${v}::text`;
|
|
3719
4093
|
return column.kind === "crossRef" && typeof v === "number" ? sql`${String(v)}::text` : void 0;
|
|
3720
4094
|
case "number": return typeof v === "number" ? sql`${v}` : void 0;
|
|
4095
|
+
case "int64": return typeof v === "bigint" ? sql`${v.toString()}::bigint` : void 0;
|
|
3721
4096
|
case "date": return typeof v === "string" || v instanceof Date ? tsParam(sql, v) : void 0;
|
|
3722
4097
|
case "boolean": return typeof v === "boolean" ? sql.unsafe(v ? "TRUE" : "FALSE") : void 0;
|
|
3723
4098
|
}
|
|
@@ -3749,12 +4124,12 @@ function buildColumnPredicate(sql, field, table, column, predicate, alias) {
|
|
|
3749
4124
|
const fit = values.filter(fits);
|
|
3750
4125
|
const misfit = values.filter((v) => !fits(v));
|
|
3751
4126
|
if (fit.length === 0) return misfit.length === 0 ? sql`FALSE` : jsonAnyOf(misfit);
|
|
3752
|
-
const list = as === "date" ? tsArrayParam(sql, fit) : as === "number" ? sql`${fit}::numeric[]` : as === "boolean" ? sql`${fit}::boolean[]` : sql`${fit.map(String)}::text[]`;
|
|
4127
|
+
const list = as === "date" ? tsArrayParam(sql, fit) : as === "number" ? sql`${fit}::numeric[]` : as === "int64" ? sql`${fit.map(String)}::text[]::bigint[]` : as === "boolean" ? sql`${fit}::boolean[]` : sql`${fit.map(String)}::text[]`;
|
|
3753
4128
|
const any = guarded(sql`${expr} = ANY(${list})`);
|
|
3754
4129
|
return misfit.length === 0 ? any : sql`(${any} OR ${jsonAnyOf(misfit)})`;
|
|
3755
4130
|
};
|
|
3756
4131
|
const ordered = (op, v) => {
|
|
3757
|
-
if (typeof v !== "number" && typeof v !== "string" && !(v instanceof Date)) throw new UnsupportedFilterError(field, op, `cannot compare against a ${v === null ? "null" : typeof v} value`);
|
|
4132
|
+
if (typeof v !== "number" && typeof v !== "bigint" && typeof v !== "string" && !(v instanceof Date)) throw new UnsupportedFilterError(field, op, `cannot compare against a ${v === null ? "null" : typeof v} value`);
|
|
3758
4133
|
return param(v);
|
|
3759
4134
|
};
|
|
3760
4135
|
const fragments = [];
|
|
@@ -3816,6 +4191,10 @@ function buildColumnPredicate(sql, field, table, column, predicate, alias) {
|
|
|
3816
4191
|
fragments.push(sql`${fieldExpr(sql, table, field, "rawText", alias)} ILIKE ${pattern}`);
|
|
3817
4192
|
break;
|
|
3818
4193
|
}
|
|
4194
|
+
case "like":
|
|
4195
|
+
case "notLike":
|
|
4196
|
+
fragments.push(as !== "text" && as !== "date" ? sql`FALSE` : likeFragment(sql, fieldExpr(sql, table, field, "rawText", alias), field, op, operand));
|
|
4197
|
+
break;
|
|
3819
4198
|
case "has":
|
|
3820
4199
|
fragments.push(sql`FALSE`);
|
|
3821
4200
|
break;
|
|
@@ -3824,6 +4203,26 @@ function buildColumnPredicate(sql, field, table, column, predicate, alias) {
|
|
|
3824
4203
|
if (fragments.length === 0) return null;
|
|
3825
4204
|
return combineAnd(sql, fragments);
|
|
3826
4205
|
}
|
|
4206
|
+
/**
|
|
4207
|
+
* An int64 column's predicate with every operand read as a bigint, as the
|
|
4208
|
+
* memory evaluator reads them. An operand that is not an int64 is refused.
|
|
4209
|
+
*/
|
|
4210
|
+
function int64Operands(field, predicate) {
|
|
4211
|
+
const one = (op, v) => {
|
|
4212
|
+
if (v === null) return v;
|
|
4213
|
+
const n = parseInt64(v);
|
|
4214
|
+
if (n === void 0) throw new UnsupportedFilterError(field, op, "requires an int64 operand");
|
|
4215
|
+
return n;
|
|
4216
|
+
};
|
|
4217
|
+
return Object.fromEntries(Object.entries(predicate).map(([op, v]) => [op, Array.isArray(v) && (op === "in" || op === "nin" || op === "between") ? v.map((e) => one(op, e)) : [
|
|
4218
|
+
"eq",
|
|
4219
|
+
"neq",
|
|
4220
|
+
"gt",
|
|
4221
|
+
"gte",
|
|
4222
|
+
"lt",
|
|
4223
|
+
"lte"
|
|
4224
|
+
].includes(op) ? one(op, v) : v]));
|
|
4225
|
+
}
|
|
3827
4226
|
|
|
3828
4227
|
//#endregion
|
|
3829
4228
|
//#region ../../packages/substrate-dataset-pg/src/pg-sql/where.ts
|
|
@@ -3832,6 +4231,13 @@ function buildColumnPredicate(sql, field, table, column, predicate, alias) {
|
|
|
3832
4231
|
* dispatch (id / shortId / metadata / promoted-column / body-path), the
|
|
3833
4232
|
* operator-fragment compiler, and the `conditionalUpdate` match fragment.
|
|
3834
4233
|
*/
|
|
4234
|
+
const CAST_TYPE = {
|
|
4235
|
+
numeric: "number",
|
|
4236
|
+
bigint: "int64",
|
|
4237
|
+
timestamptz: "date",
|
|
4238
|
+
boolean: "boolean",
|
|
4239
|
+
text: "string"
|
|
4240
|
+
};
|
|
3835
4241
|
/**
|
|
3836
4242
|
* The WHERE fragment for `where`, or null when it constrains nothing. With a
|
|
3837
4243
|
* `scope`, expressions are qualified by `scope.alias` and the walk grammar
|
|
@@ -3863,6 +4269,14 @@ function buildWhere(sql, where, castFor, table, scope) {
|
|
|
3863
4269
|
} else if (k === "NOT") {
|
|
3864
4270
|
const inner = buildWhere(sql, child, castFor, table, scope);
|
|
3865
4271
|
if (inner !== null) fragments.push(sql`NOT (${inner})`);
|
|
4272
|
+
} else if (k === "$expr") {
|
|
4273
|
+
const fieldType = (name) => table.column(name) === void 0 ? CAST_TYPE[castFor(name) ?? "text"] : void 0;
|
|
4274
|
+
const expr = compileExpr(sql, child, {
|
|
4275
|
+
table,
|
|
4276
|
+
...scope === void 0 ? {} : { alias: scope.alias },
|
|
4277
|
+
fieldType
|
|
4278
|
+
});
|
|
4279
|
+
fragments.push(sql`COALESCE(${expr}, FALSE)`);
|
|
3866
4280
|
} else if (k === "_walk") {
|
|
3867
4281
|
if (scope === void 0) throw new ResolverFrameError(WALK_UNSUPPORTED_REF, `'_walk' is not supported here`);
|
|
3868
4282
|
fragments.push(buildWalk(sql, child, table, scope));
|
|
@@ -3921,7 +4335,7 @@ function buildFieldPredicateForPath(sql, path, predicate, castFor, alias, jsonb)
|
|
|
3921
4335
|
}
|
|
3922
4336
|
const declaredCast = path.length === 1 ? castFor(field) : null;
|
|
3923
4337
|
const fragments = [];
|
|
3924
|
-
for (const [op, operand] of entries) {
|
|
4338
|
+
for (const [op, operand] of declaredCast === "bigint" ? Object.entries(int64Operands(field, predicate)) : entries) {
|
|
3925
4339
|
const frag = buildOpFragment(sql, path, op, operand, declaredCast, alias, jsonb);
|
|
3926
4340
|
if (frag !== null) fragments.push(frag);
|
|
3927
4341
|
}
|
|
@@ -3946,6 +4360,8 @@ const KNOWN_BODY_OPS = /* @__PURE__ */ new Set([
|
|
|
3946
4360
|
"contains",
|
|
3947
4361
|
"startsWith",
|
|
3948
4362
|
"endsWith",
|
|
4363
|
+
"like",
|
|
4364
|
+
"notLike",
|
|
3949
4365
|
"has"
|
|
3950
4366
|
]);
|
|
3951
4367
|
/** Recurse into a struct sub-clause, carrying the accumulated body path. */
|
|
@@ -3989,6 +4405,10 @@ function buildIdPredicate(sql, predicate, alias) {
|
|
|
3989
4405
|
case "endsWith":
|
|
3990
4406
|
if (typeof operand === "string") frag = sql`${id} ILIKE ${`%${escapeLike(operand)}`}`;
|
|
3991
4407
|
break;
|
|
4408
|
+
case "like":
|
|
4409
|
+
case "notLike":
|
|
4410
|
+
frag = likeFragment(sql, id, "id", op, operand);
|
|
4411
|
+
break;
|
|
3992
4412
|
case "gt":
|
|
3993
4413
|
frag = sql`${id} > ${metaStr("id", op, operand)}`;
|
|
3994
4414
|
break;
|
|
@@ -4077,12 +4497,15 @@ function buildOpFragment(sql, path, op, operand, declaredCast, alias, jsonb) {
|
|
|
4077
4497
|
const field = path.join(".");
|
|
4078
4498
|
const textExpr = bodyText(sql, path, alias, jsonb);
|
|
4079
4499
|
const num = bodyNum(sql, path, alias, jsonb);
|
|
4500
|
+
const i64 = bodyInt64(sql, path, alias, jsonb);
|
|
4080
4501
|
const ts = bodyTs(sql, path, alias, jsonb);
|
|
4081
4502
|
const isTs = (v) => v instanceof Date || declaredCast === "timestamptz" && typeof v === "string";
|
|
4082
4503
|
const anyOf = (values) => {
|
|
4083
4504
|
const nums = values.filter((v) => typeof v === "number");
|
|
4084
|
-
const
|
|
4505
|
+
const bigs = values.filter((v) => typeof v === "bigint");
|
|
4506
|
+
const rest = values.filter((v) => typeof v !== "number" && typeof v !== "bigint");
|
|
4085
4507
|
const parts = [];
|
|
4508
|
+
if (bigs.length > 0) parts.push(notNullAnd(sql, i64, sql`${i64} = ANY(${bigs.map(String)}::text[]::bigint[])`));
|
|
4086
4509
|
if (nums.length > 0) parts.push(notNullAnd(sql, num, sql`${num} = ANY(${nums})`));
|
|
4087
4510
|
if (rest.length > 0) parts.push(isTs(rest[0]) ? notNullAnd(sql, ts, sql`${ts} = ANY(${tsArrayParam(sql, rest)})`) : notNullAnd(sql, textExpr, sql`${textExpr} = ANY(${rest.map(scalarParam)})`));
|
|
4088
4511
|
return combineOr(sql, parts);
|
|
@@ -4092,12 +4515,14 @@ function buildOpFragment(sql, path, op, operand, declaredCast, alias, jsonb) {
|
|
|
4092
4515
|
if (operand === null) return sql`${textExpr} IS NULL`;
|
|
4093
4516
|
if (typeof operand === "boolean") return sql`${textExpr} = ${sql.unsafe(operand ? "'true'" : "'false'")}`;
|
|
4094
4517
|
if (typeof operand === "number") return notNullAnd(sql, num, sql`${num} = ${operand}`);
|
|
4518
|
+
if (typeof operand === "bigint") return notNullAnd(sql, i64, sql`${i64} = ${operand.toString()}::bigint`);
|
|
4095
4519
|
if (isTs(operand)) return notNullAnd(sql, ts, sql`${ts} = ${tsParam(sql, operand)}`);
|
|
4096
4520
|
return sql`${textExpr} = ${scalarParam(operand)}`;
|
|
4097
4521
|
case "neq":
|
|
4098
4522
|
if (operand === null) return sql`${textExpr} IS NOT NULL`;
|
|
4099
4523
|
if (typeof operand === "boolean") return sql`${textExpr} IS DISTINCT FROM ${sql.unsafe(operand ? "'true'" : "'false'")}`;
|
|
4100
4524
|
if (typeof operand === "number") return sql`NOT ${notNullAnd(sql, num, sql`${num} = ${operand}`)}`;
|
|
4525
|
+
if (typeof operand === "bigint") return sql`NOT ${notNullAnd(sql, i64, sql`${i64} = ${operand.toString()}::bigint`)}`;
|
|
4101
4526
|
if (isTs(operand)) return sql`${ts} IS DISTINCT FROM ${tsParam(sql, operand)}`;
|
|
4102
4527
|
return sql`${textExpr} IS DISTINCT FROM ${scalarParam(operand)}`;
|
|
4103
4528
|
case "gt": return rangeCmp(sql, path, operand, ">", declaredCast, alias, jsonb);
|
|
@@ -4109,6 +4534,7 @@ function buildOpFragment(sql, path, op, operand, declaredCast, alias, jsonb) {
|
|
|
4109
4534
|
const low = operand[0];
|
|
4110
4535
|
const high = operand[1];
|
|
4111
4536
|
if (typeof low === "number" && typeof high === "number") return notNullAnd(sql, num, sql`${num} BETWEEN ${low} AND ${high}`);
|
|
4537
|
+
if (typeof low === "bigint" && typeof high === "bigint") return notNullAnd(sql, i64, sql`${i64} BETWEEN ${low.toString()}::bigint AND ${high.toString()}::bigint`);
|
|
4112
4538
|
if (isTs(low) || isTs(high)) return notNullAnd(sql, ts, sql`${ts} BETWEEN ${tsParam(sql, low)} AND ${tsParam(sql, high)}`);
|
|
4113
4539
|
return sql`${textExpr} BETWEEN ${scalarParam(low)} AND ${scalarParam(high)}`;
|
|
4114
4540
|
}
|
|
@@ -4137,6 +4563,8 @@ function buildOpFragment(sql, path, op, operand, declaredCast, alias, jsonb) {
|
|
|
4137
4563
|
const pattern = op === "contains" ? `%${e}%` : op === "startsWith" ? `${e}%` : `%${e}`;
|
|
4138
4564
|
return sql`${bodyRawText(sql, path, alias, jsonb)} ILIKE ${pattern}`;
|
|
4139
4565
|
}
|
|
4566
|
+
case "like":
|
|
4567
|
+
case "notLike": return likeFragment(sql, bodyRawText(sql, path, alias, jsonb), field, op, operand);
|
|
4140
4568
|
case "has": {
|
|
4141
4569
|
const elem = operand instanceof Date ? operand.toISOString() : operand;
|
|
4142
4570
|
return sql`${bodyJson(sql, path, alias, jsonb)} @> ${sql.json(asJsonValue([elem]))}`;
|
|
@@ -4163,6 +4591,10 @@ function rangeCmp(sql, path, operand, cmp, declaredCast, alias, jsonb) {
|
|
|
4163
4591
|
const num = bodyNum(sql, path, alias, jsonb);
|
|
4164
4592
|
return notNullAnd(sql, num, compare(num, operand));
|
|
4165
4593
|
}
|
|
4594
|
+
if (typeof operand === "bigint") {
|
|
4595
|
+
const i64 = bodyInt64(sql, path, alias, jsonb);
|
|
4596
|
+
return notNullAnd(sql, i64, compare(i64, sql`${operand.toString()}::bigint`));
|
|
4597
|
+
}
|
|
4166
4598
|
if (operand instanceof Date || declaredCast === "timestamptz" && typeof operand === "string") {
|
|
4167
4599
|
const ts = bodyTs(sql, path, alias, jsonb);
|
|
4168
4600
|
return notNullAnd(sql, ts, compare(ts, tsParam(sql, operand)));
|
|
@@ -4182,7 +4614,7 @@ function conditionalMatchFragment(sql, field, expected, table) {
|
|
|
4182
4614
|
const v = fieldExpr(sql, table, field, "json");
|
|
4183
4615
|
if (expected === null || expected === void 0) return sql`(${v} IS NULL OR ${v} = 'null'::jsonb)`;
|
|
4184
4616
|
if (expected instanceof Date) return sql`${fieldExpr(sql, table, field, "date")} = ${tsParam(sql, expected)}`;
|
|
4185
|
-
return sql`${v} = ${JSON.stringify(expected)}::text::jsonb`;
|
|
4617
|
+
return sql`${v} = ${typeof expected === "bigint" ? expected.toString() : JSON.stringify(expected)}::text::jsonb`;
|
|
4186
4618
|
}
|
|
4187
4619
|
const textExpr = bodyText(sql, [field]);
|
|
4188
4620
|
if (expected === null || expected === void 0) return sql`${textExpr} IS NULL`;
|
|
@@ -4558,6 +4990,7 @@ function sortKey(sql, table, field, cast, alias) {
|
|
|
4558
4990
|
const column = table.column(field);
|
|
4559
4991
|
if (column !== void 0) return columnOrderExpr(sql, table, column, alias);
|
|
4560
4992
|
if (cast === "numeric") return bodyNum(sql, [field], alias, table.jsonb);
|
|
4993
|
+
if (cast === "bigint") return bodyInt64(sql, [field], alias, table.jsonb);
|
|
4561
4994
|
if (cast === "timestamptz") return bodyTs(sql, [field], alias, table.jsonb);
|
|
4562
4995
|
return bodyText(sql, [field], alias, table.jsonb);
|
|
4563
4996
|
}
|
|
@@ -4683,27 +5116,6 @@ function refOperand(sql, field, op, ref, kind, scope) {
|
|
|
4683
5116
|
const micros = ref.plus === void 0 ? 0 : Math.round(ref.plus * 1e3);
|
|
4684
5117
|
return micros === 0 ? right : sql`(${right} + ${sql.unsafe(`(${micros})`)} * interval '1 microsecond')`;
|
|
4685
5118
|
}
|
|
4686
|
-
/**
|
|
4687
|
-
* A `params.derive` value for the seed row, as float8: `diff` is a − b and
|
|
4688
|
-
* `ratio` a / b, NULL for a null or non-numeric operand or a zero divisor, as
|
|
4689
|
-
* memory's; `value` is its one operand as it is. Operands are `$.<field>` or `_agg.<name>`, read as `$ref`s are.
|
|
4690
|
-
*
|
|
4691
|
-
* As memory's: a magnitude outside [1e-300, 1e300] is NULL, and 0 stays only
|
|
4692
|
-
* when exact (x = y, or a zero numerator). float8 raises on overflow and on a
|
|
4693
|
-
* division that underflows to 0, so the guard computes only where neither can
|
|
4694
|
-
* happen (operands within 1e307 or of one sign; a log ratio within ±692, which
|
|
4695
|
-
* is ±(ln 1e300 + 1.2)), and anything it skips lies outside the range anyway.
|
|
4696
|
-
* One expression, so ORDER BY, the cursor compare and the null tail share it.
|
|
4697
|
-
*/
|
|
4698
|
-
function buildDerive(sql, name, spec, scope) {
|
|
4699
|
-
if ("value" in spec) return refOperand(sql, `$${name}`, "value", { $ref: spec.value }, "number", scope);
|
|
4700
|
-
const [op, [a, b]] = Object.entries(spec)[0];
|
|
4701
|
-
const operand = (r) => refOperand(sql, `$${name}`, op, { $ref: r }, "number", scope);
|
|
4702
|
-
return sql`(SELECT ${op === "diff" ? sql.unsafe(`CASE WHEN sign(x) * sign(y) >= 0 OR greatest(abs(x), abs(y)) <= 1e307
|
|
4703
|
-
THEN CASE WHEN x = y OR abs(x - y) BETWEEN 1e-300 AND 1e300 THEN x - y END END`) : sql.unsafe(`CASE WHEN y = 0 THEN NULL WHEN x = 0 THEN x / y
|
|
4704
|
-
WHEN abs(ln(abs(x)) - ln(abs(y))) <= 692
|
|
4705
|
-
THEN CASE WHEN abs(x / y) BETWEEN 1e-300 AND 1e300 THEN x / y END END`)} FROM (SELECT ${operand(a)} AS x, ${operand(b)} AS y OFFSET 0) d)`;
|
|
4706
|
-
}
|
|
4707
5119
|
const f64 = /* @__PURE__ */ new Float64Array(1);
|
|
4708
5120
|
const u64 = new BigUint64Array(f64.buffer);
|
|
4709
5121
|
function bitsOf(x) {
|
|
@@ -4791,7 +5203,7 @@ function aggOver(sql, spec, table, alias, fieldDef) {
|
|
|
4791
5203
|
agg: sql`to_jsonb(count(*))`,
|
|
4792
5204
|
kind: "number"
|
|
4793
5205
|
};
|
|
4794
|
-
if ("latest" in spec
|
|
5206
|
+
if ("latest" in spec) {
|
|
4795
5207
|
const { by, value } = spec.latest;
|
|
4796
5208
|
const key = sortKey(sql, table, by, resolveBodyOrderByCast(fieldDef(by)), alias);
|
|
4797
5209
|
const numeric = value === "seq" || resolveBodyOrderByCast(fieldDef(value)) === "numeric";
|
|
@@ -4826,19 +5238,22 @@ function aggOver(sql, spec, table, alias, fieldDef) {
|
|
|
4826
5238
|
agg: sql`to_jsonb(${sql.unsafe(op)}(${n}))`,
|
|
4827
5239
|
kind: "number"
|
|
4828
5240
|
};
|
|
4829
|
-
const
|
|
4830
|
-
const v = isMetadataField(field) ? sql.unsafe(`${metadataColumnSql(field, alias)}::numeric`) : c === void 0 || extraWins(c) ? fieldExpr(sql, table, field, "number", alias) : sql`${fieldExpr(sql, table, field, "number", alias)}::text::numeric`;
|
|
4831
|
-
const s = sql`${sql.unsafe(op)}(${v})`;
|
|
5241
|
+
const s = sql`${sql.unsafe(op)}(${exactNumeric(sql, table, field, alias)})`;
|
|
4832
5242
|
return {
|
|
4833
|
-
agg: sql`to_jsonb(CASE WHEN abs(${s}) >= ${sql.unsafe(OVERFLOW)} THEN NULL
|
|
4834
|
-
WHEN abs(${s}) <= ${sql.unsafe(UNDERFLOW)} THEN 0::float8 ELSE ${s}::float8 END)`,
|
|
5243
|
+
agg: sql`to_jsonb(CASE WHEN abs(${s}) >= ${sql.unsafe(OVERFLOW$1)} THEN NULL
|
|
5244
|
+
WHEN abs(${s}) <= ${sql.unsafe(UNDERFLOW$1)} THEN 0::float8 ELSE ${s}::float8 END)`,
|
|
4835
5245
|
kind: "number"
|
|
4836
5246
|
};
|
|
4837
5247
|
}
|
|
5248
|
+
/** A number field read as an exact numeric: a stored decimal is never narrowed through float8. */
|
|
5249
|
+
function exactNumeric(sql, table, field, alias) {
|
|
5250
|
+
const c = table.column(field);
|
|
5251
|
+
return isMetadataField(field) ? sql.unsafe(`${metadataColumnSql(field, alias)}::numeric`) : c === void 0 || extraWins(c) ? fieldExpr(sql, table, field, "number", alias) : sql`${fieldExpr(sql, table, field, "number", alias)}::text::numeric`;
|
|
5252
|
+
}
|
|
4838
5253
|
/** 2^1024 − 2^970, the least value float8 rounds to Infinity, as an exact numeric. */
|
|
4839
|
-
const OVERFLOW = `${2n ** 1024n - 2n ** 970n}::numeric`;
|
|
5254
|
+
const OVERFLOW$1 = `${2n ** 1024n - 2n ** 970n}::numeric`;
|
|
4840
5255
|
/** 2^-1075 (= 5^1075 / 10^1075), the greatest value float8 rounds to 0, as an exact numeric. */
|
|
4841
|
-
const UNDERFLOW = `0.${(5n ** 1075n).toString().padStart(1075, "0")}::numeric`;
|
|
5256
|
+
const UNDERFLOW$1 = `0.${(5n ** 1075n).toString().padStart(1075, "0")}::numeric`;
|
|
4842
5257
|
/**
|
|
4843
5258
|
* `p` over `table`'s rows aliased `alias` passing `where`: `ORDER BY key, id LIMIT 1`,
|
|
4844
5259
|
* a top-1 an index can serve, never a sort of every row. An index keyed the
|
|
@@ -4987,46 +5402,6 @@ function buildFrameCount(sql, where, scope, joins = sql``) {
|
|
|
4987
5402
|
return sql`SELECT count(*)::float8 AS n FROM ${sql.unsafe(from)} ${sql.unsafe(scope.alias)}${joins} WHERE ${where}`;
|
|
4988
5403
|
}
|
|
4989
5404
|
/**
|
|
4990
|
-
* A whole-result aggregate over `scope`'s rows matching `where`, one row
|
|
4991
|
-
* `{ g }` per group: one group, or one per UTC day/hour of `bucket.field`
|
|
4992
|
-
* ascending, rows with no readable date in none, at most `limit` groups.
|
|
4993
|
-
* A pick reads the whole result through `pickRow`, shadowing the outer
|
|
4994
|
-
* alias; in a bucket it runs in two phases: GROUP BY finds the winning key,
|
|
4995
|
-
* then a top-1 on `key = it` (the indexed expression; the bucket key only
|
|
4996
|
-
* pins the lookup, never the frame's WHERE) fetches that row's stored value.
|
|
4997
|
-
*/
|
|
4998
|
-
function buildFrameAggregate(sql, aggregate, bucket, where, limit, scope) {
|
|
4999
|
-
const r = scope.alias;
|
|
5000
|
-
const R = sql.unsafe(r);
|
|
5001
|
-
const table = scope.tableFor(scope.type);
|
|
5002
|
-
const T = sql.unsafe(table.from);
|
|
5003
|
-
const fieldDef = (f) => scope.fieldDef(scope.type, f);
|
|
5004
|
-
const aggs = [];
|
|
5005
|
-
const picks = [];
|
|
5006
|
-
for (const [name, spec] of Object.entries(aggregate)) {
|
|
5007
|
-
const { agg, pick } = aggOver(sql, spec, table, r, fieldDef);
|
|
5008
|
-
if (pick === void 0) aggs.push(sql`${name}::text, ${agg}`);
|
|
5009
|
-
else picks.push([name, pick]);
|
|
5010
|
-
}
|
|
5011
|
-
if (bucket === void 0) {
|
|
5012
|
-
const picked = picks.map(([name, p]) => sql`${name}::text, ${pickRow(sql, p, table, r, where)}`);
|
|
5013
|
-
return sql`SELECT ${aggs.length === 0 ? sql`'{}'::jsonb` : sql`(SELECT ${buildObject(sql, aggs)} FROM ${T} ${R} WHERE ${where})`} || ${buildObject(sql, picked)} AS g`;
|
|
5014
|
-
}
|
|
5015
|
-
const [by, fmt] = bucket.by === "day" ? ["day", "YYYY-MM-DD"] : ["hour", "YYYY-MM-DD\"T\"HH24"];
|
|
5016
|
-
const key = sql`to_char(date_trunc(${sql.unsafe(`'${by}'`)}, ${refValue(sql, table, bucket.field, "date", r)} AT TIME ZONE 'UTC'), ${sql.unsafe(`'${fmt}'`)})`;
|
|
5017
|
-
const winners = picks.reduce((acc, [, p], i) => sql`${acc}, ${sql.unsafe(p.dir === "ASC" ? "min" : "max")}(${p.key}) AS ${sql.unsafe(`m${i}`)}`, sql``);
|
|
5018
|
-
const picked = picks.map(([name, p], i) => {
|
|
5019
|
-
const at = combineAnd(sql, [
|
|
5020
|
-
where,
|
|
5021
|
-
sql`${p.key} = ${sql.unsafe(`x.m${i}`)}`,
|
|
5022
|
-
sql`${key} = x.b`
|
|
5023
|
-
]);
|
|
5024
|
-
return sql`${name}::text, ${pickRow(sql, p, table, r, at)}`;
|
|
5025
|
-
});
|
|
5026
|
-
const cut = limit === void 0 ? sql`` : sql`LIMIT ${limit + 1}`;
|
|
5027
|
-
return sql`SELECT jsonb_build_object('bucket', x.b) || x.g || ${buildObject(sql, picked)} AS g FROM (SELECT ${key} AS b, ${buildObject(sql, aggs)} AS g${winners} FROM ${T} ${R} WHERE ${where} GROUP BY 1) x WHERE x.b IS NOT NULL ORDER BY x.b COLLATE "C" ${cut}`;
|
|
5028
|
-
}
|
|
5029
|
-
/**
|
|
5030
5405
|
* `{ _agg, <cmp>: operand }`: never NULL, false for a null aggregate or a
|
|
5031
5406
|
* null operand. Dates compare as instants; a number operand (or any operand
|
|
5032
5407
|
* of a number aggregate) as float8; anything else as stored, eq/neq by jsonb
|
|
@@ -5086,11 +5461,91 @@ function buildWalkProjection(sql, aggs, nested, table, scope) {
|
|
|
5086
5461
|
return sql`(SELECT ${buildObject(sql, pairs)}${truncated} FROM (SELECT ${list} OFFSET 0) ${sql.unsafe(x)})`;
|
|
5087
5462
|
}
|
|
5088
5463
|
/**
|
|
5089
|
-
*
|
|
5090
|
-
*
|
|
5091
|
-
*
|
|
5464
|
+
* A row's top-level walk selections as their own SELECT items, beside `extra`:
|
|
5465
|
+
* `_agg` as one jsonb object, each nested list as a json array of up to
|
|
5466
|
+
* `limit + 1` rows. {@link mergeWalkColumns} cuts the lists and merges them
|
|
5467
|
+
* into `extra` as {@link buildWalkProjection} does in SQL, which costs pg far
|
|
5468
|
+
* more per row on a large read. Null when there are none.
|
|
5469
|
+
*/
|
|
5470
|
+
function buildWalkColumns(sql, aggs, nested, table, scope) {
|
|
5471
|
+
const columns = [];
|
|
5472
|
+
const items = [];
|
|
5473
|
+
if (aggs !== void 0) {
|
|
5474
|
+
columns.push({
|
|
5475
|
+
name: "_agg",
|
|
5476
|
+
column: "__walk_agg"
|
|
5477
|
+
});
|
|
5478
|
+
items.push(sql`${buildWalkAggs(sql, aggs, table, scope)} AS __walk_agg`);
|
|
5479
|
+
}
|
|
5480
|
+
for (const [i, [name, sel]] of Object.entries(nested).entries()) {
|
|
5481
|
+
const column = `__walk_${i}`;
|
|
5482
|
+
columns.push(sel.limit === void 0 ? {
|
|
5483
|
+
name,
|
|
5484
|
+
column
|
|
5485
|
+
} : {
|
|
5486
|
+
name,
|
|
5487
|
+
column,
|
|
5488
|
+
limit: sel.limit
|
|
5489
|
+
});
|
|
5490
|
+
items.push(sql`${buildNested(sql, sel, table, scope, "json")} AS ${sql.unsafe(column)}`);
|
|
5491
|
+
}
|
|
5492
|
+
if (items.length === 0) return null;
|
|
5493
|
+
return {
|
|
5494
|
+
select: sql`${items.flatMap((f, i) => i === 0 ? [f] : [sql`,`, f])}`,
|
|
5495
|
+
columns
|
|
5496
|
+
};
|
|
5497
|
+
}
|
|
5498
|
+
/**
|
|
5499
|
+
* `record` with its walk columns merged into `extra` as SQL's `extra || walk`
|
|
5500
|
+
* would: each list cut to its limit, `_truncated` naming the lists that had
|
|
5501
|
+
* more, the walk's keys winning, and every key in jsonb's order (shorter
|
|
5502
|
+
* first, then bytewise), so `decodeRow` reads the row it did before. Pass one
|
|
5503
|
+
* `orders` map per read: it caches the sorted order of each key set seen.
|
|
5092
5504
|
*/
|
|
5093
|
-
function
|
|
5505
|
+
function mergeWalkColumns(record, columns, orders = /* @__PURE__ */ new Map()) {
|
|
5506
|
+
const extra = record.extra ?? {};
|
|
5507
|
+
const walk = {};
|
|
5508
|
+
const cut = {};
|
|
5509
|
+
let anyCut = false;
|
|
5510
|
+
for (const c of columns) {
|
|
5511
|
+
const v = record[c.column];
|
|
5512
|
+
if (c.limit !== void 0 && Array.isArray(v) && v.length > c.limit) {
|
|
5513
|
+
walk[c.name] = v.slice(0, c.limit);
|
|
5514
|
+
cut[c.name] = true;
|
|
5515
|
+
anyCut = true;
|
|
5516
|
+
} else walk[c.name] = v;
|
|
5517
|
+
}
|
|
5518
|
+
if (anyCut) walk._truncated = jsonbOrdered(cut);
|
|
5519
|
+
const extraKeys = Object.keys(extra);
|
|
5520
|
+
const sig = `${anyCut ? 1 : 0}${extraKeys.join("\0")}`;
|
|
5521
|
+
let order = orders.get(sig);
|
|
5522
|
+
if (order === void 0) {
|
|
5523
|
+
order = [.../* @__PURE__ */ new Set([...extraKeys, ...Object.keys(walk)])].sort(jsonbKeyOrder);
|
|
5524
|
+
orders.set(sig, order);
|
|
5525
|
+
}
|
|
5526
|
+
const merged = {};
|
|
5527
|
+
for (const k of order) merged[k] = Object.hasOwn(walk, k) ? walk[k] : extra[k];
|
|
5528
|
+
record.extra = merged;
|
|
5529
|
+
return record;
|
|
5530
|
+
}
|
|
5531
|
+
/** jsonb's object key order: shorter (in UTF-8 bytes) first, then bytewise. */
|
|
5532
|
+
function jsonbKeyOrder(a, b) {
|
|
5533
|
+
if (Buffer.byteLength(a) === a.length && Buffer.byteLength(b) === b.length) return a.length - b.length || (a < b ? -1 : a > b ? 1 : 0);
|
|
5534
|
+
const x = Buffer.from(a);
|
|
5535
|
+
const y = Buffer.from(b);
|
|
5536
|
+
return x.length - y.length || Buffer.compare(x, y);
|
|
5537
|
+
}
|
|
5538
|
+
function jsonbOrdered(o) {
|
|
5539
|
+
const out = {};
|
|
5540
|
+
for (const k of Object.keys(o).sort(jsonbKeyOrder)) out[k] = o[k];
|
|
5541
|
+
return out;
|
|
5542
|
+
}
|
|
5543
|
+
/**
|
|
5544
|
+
* One nested selection for `scope`'s row: a jsonb array (or, with `as` json, a
|
|
5545
|
+
* json array of jsonb objects) of up to `limit + 1` far rows, each `id`, its
|
|
5546
|
+
* selected fields (null when absent) and its own walk selections; no `_metadata`.
|
|
5547
|
+
*/
|
|
5548
|
+
function buildNested(sql, sel, table, scope, as = "jsonb") {
|
|
5094
5549
|
const alias = scope.nextAlias();
|
|
5095
5550
|
const a = sql.unsafe(alias);
|
|
5096
5551
|
const { type, from, cond } = hop(sql, sel._walk, scope, alias, table);
|
|
@@ -5114,12 +5569,177 @@ function buildNested(sql, sel, table, scope) {
|
|
|
5114
5569
|
const id = sql.unsafe(`CASE WHEN jsonb_typeof(${doc} -> 'id') IN ('number', 'string') THEN ${doc} -> 'id' ELSE to_jsonb(${alias}.id) END`);
|
|
5115
5570
|
const obj = buildObject(sql, [sql`'id'::text, ${id}`, ...plain.map((k) => sql`${k}::text, ${fieldExpr(sql, farTable, k, "json", alias)}`)]);
|
|
5116
5571
|
const sub = buildWalkProjection(sql, subAggs, subNested, farTable, farScope);
|
|
5117
|
-
|
|
5572
|
+
const row = sub === null ? obj : sql`${obj} || ${sub}`;
|
|
5573
|
+
return sql`(SELECT COALESCE(${sql.unsafe(as === "json" ? "json_agg" : "jsonb_agg")}(${row} ORDER BY ${orderBy}), ${sql.unsafe(`'[]'::${as}`)}) FROM (SELECT ${a}.* FROM ${sql.unsafe(from)} ${a} WHERE ${combineAnd(sql, [cond, ...filters])} ORDER BY ${orderBy} ${limit}) ${a})`;
|
|
5574
|
+
}
|
|
5575
|
+
|
|
5576
|
+
//#endregion
|
|
5577
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-sql/grouped.ts
|
|
5578
|
+
/**
|
|
5579
|
+
* A grouped read as one statement:
|
|
5580
|
+
*
|
|
5581
|
+
* SELECT … FROM (SELECT <keys>, <aggs> FROM T r JOIN… WHERE … GROUP BY 1, 2…) x
|
|
5582
|
+
* WHERE <having> ORDER BY … LIMIT limit+1 OFFSET offset
|
|
5583
|
+
*
|
|
5584
|
+
* The inner select's `GROUP BY` names the keys by ordinal and everything after
|
|
5585
|
+
* it reads them as `x.k<i>` / `x.a<j>`: a key expression is never written
|
|
5586
|
+
* twice, because each copy of a literal is its own bind parameter and Postgres
|
|
5587
|
+
* would then not see them as one expression. The memory reference is
|
|
5588
|
+
* `@mnmal-ai/substrate-filter`'s `groupRows`.
|
|
5589
|
+
*/
|
|
5590
|
+
/** 2^1024 − 2^970, the least value float8 rounds to Infinity, as an exact numeric. */
|
|
5591
|
+
const OVERFLOW = `${2n ** 1024n - 2n ** 970n}::numeric`;
|
|
5592
|
+
/** 2^-1075, the greatest value float8 rounds to 0, as an exact numeric. */
|
|
5593
|
+
const UNDERFLOW = `0.${(5n ** 1075n).toString().padStart(1075, "0")}::numeric`;
|
|
5594
|
+
const MICROS = `'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'`;
|
|
5595
|
+
function buildGroupedRead(sql, spec, matched, joins, table, fieldDef) {
|
|
5596
|
+
const u = (s) => sql.unsafe(s);
|
|
5597
|
+
const fieldType = (name) => exprTypeOfField(fieldDef(name));
|
|
5598
|
+
const compile = (e) => compileExprTyped(sql, e, {
|
|
5599
|
+
table,
|
|
5600
|
+
alias: "r",
|
|
5601
|
+
fieldType
|
|
5602
|
+
});
|
|
5603
|
+
const list = (fs) => fs.reduce((a, b) => sql`${a}, ${b}`);
|
|
5604
|
+
const keys = Object.entries(spec.groupBy ?? {}).map(([name, e], i) => {
|
|
5605
|
+
const c = compile(e);
|
|
5606
|
+
if (c.type === null) throw new Error(`internal: group key '${name}' has no type`);
|
|
5607
|
+
return {
|
|
5608
|
+
name,
|
|
5609
|
+
i,
|
|
5610
|
+
frag: c.fragment,
|
|
5611
|
+
type: c.type
|
|
5612
|
+
};
|
|
5613
|
+
});
|
|
5614
|
+
const aggs = Object.entries(spec.aggregate).map(([name, a], j) => {
|
|
5615
|
+
const { op, arg } = aggParts(a);
|
|
5616
|
+
const c = arg === true && op === "count" ? void 0 : compile(arg);
|
|
5617
|
+
const t = c?.type ?? null;
|
|
5618
|
+
const f = c?.fragment;
|
|
5619
|
+
const text = typeof arg === "string" && t === "number" && metadataExprType(arg) === void 0 ? exactNumeric(sql, table, arg, "r") : sql`(${f})::text::numeric`;
|
|
5620
|
+
let raw;
|
|
5621
|
+
switch (op) {
|
|
5622
|
+
case "count":
|
|
5623
|
+
raw = c === void 0 ? sql`count(*)` : sql`count(${f})`;
|
|
5624
|
+
break;
|
|
5625
|
+
case "countDistinct":
|
|
5626
|
+
raw = sql`count(DISTINCT ${f})`;
|
|
5627
|
+
break;
|
|
5628
|
+
case "sum":
|
|
5629
|
+
raw = t === "int64" ? sql`sum(${f})::bigint` : sql`sum(${text})`;
|
|
5630
|
+
break;
|
|
5631
|
+
case "avg":
|
|
5632
|
+
raw = t === "int64" ? sql`sum(${f})::float8 / NULLIF(count(${f}), 0)::float8` : sql`avg(${text})`;
|
|
5633
|
+
break;
|
|
5634
|
+
default: raw = sql`${u(op)}(${t === "string" ? sql`(${f}) COLLATE "C"` : f})`;
|
|
5635
|
+
}
|
|
5636
|
+
const type = aggType(op, t);
|
|
5637
|
+
const a_ = u(`x.a${j}`);
|
|
5638
|
+
let typed;
|
|
5639
|
+
if (op === "count" || op === "countDistinct") typed = sql`${a_}::float8`;
|
|
5640
|
+
else if (op === "avg" || op === "sum") typed = t === "int64" ? a_ : floatOf(a_);
|
|
5641
|
+
else typed = a_;
|
|
5642
|
+
return {
|
|
5643
|
+
name,
|
|
5644
|
+
j,
|
|
5645
|
+
raw,
|
|
5646
|
+
typed,
|
|
5647
|
+
type,
|
|
5648
|
+
raises: op === "sum" && t === "int64" || canRaise(arg)
|
|
5649
|
+
};
|
|
5650
|
+
});
|
|
5651
|
+
function floatOf(s) {
|
|
5652
|
+
return sql`(CASE WHEN abs(${s}) >= ${u(OVERFLOW)} THEN NULL WHEN abs(${s}) <= ${u(UNDERFLOW)} THEN 0::float8 ELSE ${s}::float8 END)`;
|
|
5653
|
+
}
|
|
5654
|
+
const raises = aggs.some((a) => a.raises) || Object.values(spec.groupBy ?? {}).some(canRaise) || canRaise(spec.having);
|
|
5655
|
+
const grouped = sql`SELECT ${list([...keys.map((k) => sql`${k.frag} AS ${u(`k${k.i}`)}`), ...aggs.map((a) => sql`${a.raw} AS ${u(`a${a.j}`)}`)])} FROM ${u(table.from)} r${joins} WHERE ${matched}${keys.length === 0 ? sql`` : sql` GROUP BY ${u(keys.map((_, i) => i + 1).join(", "))}`} OFFSET 0`;
|
|
5656
|
+
const inner = raises ? sql`SELECT y.*, count(*) OVER () AS w FROM (${grouped}) y OFFSET 0` : grouped;
|
|
5657
|
+
const outputs = new Map([...keys.map((k) => [k.name, u(`x.k${k.i}`)]), ...aggs.map((a) => [a.name, a.typed])]);
|
|
5658
|
+
const types = new Map([...keys.map((k) => [k.name, k.type]), ...aggs.map((a) => [a.name, a.type])]);
|
|
5659
|
+
const having = spec.having === void 0 ? sql`` : sql` WHERE ${compileExprTyped(sql, spec.having, {
|
|
5660
|
+
table,
|
|
5661
|
+
alias: "r",
|
|
5662
|
+
outputs,
|
|
5663
|
+
fieldType: (n) => types.get(n)
|
|
5664
|
+
}).fragment}`;
|
|
5665
|
+
const col = new Map([...keys.map((k) => [k.name, u(`x.k${k.i}`)]), ...aggs.map((a) => [a.name, a.typed])]);
|
|
5666
|
+
const order = [...(spec.orderBy ?? []).map((c) => Object.entries(c)[0]), ...keys.map((k) => [k.name, "asc"])].map(([name, dir]) => {
|
|
5667
|
+
const c = col.get(name);
|
|
5668
|
+
if (c === void 0) throw new Error(`internal: orderBy '${name}' names no output`);
|
|
5669
|
+
return sql`${types.get(name) === "string" ? sql`${c} COLLATE "C"` : c} ${u(dir === "asc" ? "ASC" : "DESC")} NULLS LAST`;
|
|
5670
|
+
});
|
|
5671
|
+
const project = (frag, type) => type === "date" ? sql`to_char((${frag}) AT TIME ZONE 'UTC', ${u(MICROS)})` : type === "int64" ? sql`(${frag})::text` : frag;
|
|
5672
|
+
const outs = [...keys.map((k) => ({
|
|
5673
|
+
name: k.name,
|
|
5674
|
+
type: k.type,
|
|
5675
|
+
frag: u(`x.k${k.i}`)
|
|
5676
|
+
})), ...aggs.map((a) => ({
|
|
5677
|
+
name: a.name,
|
|
5678
|
+
type: a.type,
|
|
5679
|
+
frag: a.typed
|
|
5680
|
+
}))];
|
|
5681
|
+
const limit = spec.limit === void 0 ? sql`` : sql` LIMIT ${spec.limit + 1}`;
|
|
5682
|
+
const offset = spec.offset === void 0 || spec.offset === 0 ? sql`` : sql` OFFSET ${spec.offset}`;
|
|
5683
|
+
return {
|
|
5684
|
+
statement: sql`SELECT 1 AS g, ${list([...outs.map((o, n) => sql`${project(o.frag, o.type)} AS ${u(`o${n}`)}`), ...raises ? [sql`x.w`] : []])} FROM (${inner}) x${having}${order.length === 0 ? sql`` : sql` ORDER BY ${list(order)}`}${limit}${offset}`,
|
|
5685
|
+
decode: (row) => Object.fromEntries(outs.map((o, n) => {
|
|
5686
|
+
const v = row[`o${n}`] ?? null;
|
|
5687
|
+
return [o.name, o.type === "int64" && v !== null ? BigInt(v) : v];
|
|
5688
|
+
}))
|
|
5689
|
+
};
|
|
5118
5690
|
}
|
|
5119
5691
|
|
|
5120
5692
|
//#endregion
|
|
5121
5693
|
//#region ../../packages/substrate-dataset-pg/src/promote.ts
|
|
5122
5694
|
/**
|
|
5695
|
+
* The promotion sweep (`tables` layout): each `pending` field gets its column,
|
|
5696
|
+
* and each `promoting` field's values move out of `extra` into it. One field
|
|
5697
|
+
* at a time, so DDL touches one type's table, under a short `lock_timeout`.
|
|
5698
|
+
*
|
|
5699
|
+
* Per field:
|
|
5700
|
+
* 1. `pending` only: ADD COLUMN, with the type lock's wait and the table
|
|
5701
|
+
* lock's both under `lock_timeout`. On a timeout the field is `busy` and
|
|
5702
|
+
* the sweep moves on; the next sweep retries it.
|
|
5703
|
+
* 2. `pending` only: record it `promoting`, and notify.
|
|
5704
|
+
* 3. Backfill, keyset-paginated by `id` (F14): each page moves the values of
|
|
5705
|
+
* its rows that carry the key, through the row codec, so a misfit lands in
|
|
5706
|
+
* `$raw` exactly as a write would put it; and a `$raw` value the column
|
|
5707
|
+
* now admits (its enum vocabulary widened) into the column. Those rows are locked in id order
|
|
5708
|
+
* first, as batch writers lock them; a deadlock or lock timeout is `busy`.
|
|
5709
|
+
* A row a writer changed between the page's read and its update is left
|
|
5710
|
+
* for the straggler pass.
|
|
5711
|
+
* 4. Reconcile the type's indexes, which builds any waiting on the column.
|
|
5712
|
+
* 5. Record it `column` (only from `promoting`, under the type lock, so never
|
|
5713
|
+
* over a boot that orphaned or redeclared it), bump the catalog version,
|
|
5714
|
+
* notify, and run one straggler pass (step 3 again), for values a writer
|
|
5715
|
+
* put in `extra` before the move. The move locks the table SHARE, retried
|
|
5716
|
+
* NOWAIT until `lock_timeout` so writes never queue behind it (held past
|
|
5717
|
+
* it, the field is `busy`; `noWriters` queues instead), so every such write has
|
|
5718
|
+
* committed before it, and any later one fails its version check and is
|
|
5719
|
+
* rebuilt against the new table (`guardCte`).
|
|
5720
|
+
*
|
|
5721
|
+
* A unique index refusing a backfilled value (23505), or one that cannot be
|
|
5722
|
+
* built over the moved values, fails the field and is never weakened to let
|
|
5723
|
+
* the move through. Before step 5 the field stays `promoting`, so reads still
|
|
5724
|
+
* see its `extra` values. A field goes back from `column` only when its enum
|
|
5725
|
+
* vocabulary widens (`ensureTypeTable`), so step 3 moves the `$raw` values the
|
|
5726
|
+
* vocabulary now admits; otherwise never: a straggler
|
|
5727
|
+
* that fails (or a straggler pass that is stopped) leaves the field `column`
|
|
5728
|
+
* with its error recorded, which brings it back to a later sweep's straggler
|
|
5729
|
+
* pass until that pass finishes. A field that failed on a unique conflict,
|
|
5730
|
+
* `promoting` or `column`, comes back only once the type's catalog version has
|
|
5731
|
+
* moved on (`error_version`), since a rescan of the whole table at the same
|
|
5732
|
+
* version only meets it again.
|
|
5733
|
+
*
|
|
5734
|
+
* Every server process sweeps, so each field is swept under a session
|
|
5735
|
+
* advisory lock taken with a try ({@link promotionLockKey}): a field another
|
|
5736
|
+
* sweep holds is skipped, not swept twice, and reported `busy` with
|
|
5737
|
+
* {@link HELD_ELSEWHERE}. The lock never waits, so it is in no wait cycle;
|
|
5738
|
+
* it is session-level, not transaction-level, because the
|
|
5739
|
+
* field's own index builds (CONCURRENTLY) would wait forever on an open
|
|
5740
|
+
* transaction of the sweep's.
|
|
5741
|
+
*/
|
|
5742
|
+
/**
|
|
5123
5743
|
* The error of a `busy` event for a field whose sweep lock another sweep holds:
|
|
5124
5744
|
* not an attempt, and never recorded, only reported.
|
|
5125
5745
|
*/
|
|
@@ -5277,7 +5897,7 @@ async function backfill(sql, table, field, opts) {
|
|
|
5277
5897
|
if (column === void 0) throw new Error(`internal: ${table.type}.${field} has no column`);
|
|
5278
5898
|
const from = sql.unsafe(table.from);
|
|
5279
5899
|
const col = sql.unsafe(ident(column.column));
|
|
5280
|
-
const cast = sql.unsafe(PG_TYPE[column.kind]);
|
|
5900
|
+
const cast = sql.unsafe(PG_TYPE$1[column.kind]);
|
|
5281
5901
|
let last = "";
|
|
5282
5902
|
for (;;) {
|
|
5283
5903
|
if (opts.signal?.aborted === true) return false;
|
|
@@ -5295,7 +5915,8 @@ async function backfill(sql, table, field, opts) {
|
|
|
5295
5915
|
for (const r of page) {
|
|
5296
5916
|
const stored = r.v ?? r.rv;
|
|
5297
5917
|
if (stored === null) continue;
|
|
5298
|
-
const
|
|
5918
|
+
const parsed = column.kind === "int64" && /^-?\d+$/.test(stored) && parseInt64(BigInt(stored)) !== void 0 ? BigInt(stored) : JSON.parse(stored);
|
|
5919
|
+
const encoded = encodeRow(table, { [field]: parsed });
|
|
5299
5920
|
const c = encoded.columns[column.column];
|
|
5300
5921
|
const misfit = Object.hasOwn(encoded.raw, field);
|
|
5301
5922
|
if (r.v === null) {
|
|
@@ -5308,7 +5929,7 @@ async function backfill(sql, table, field, opts) {
|
|
|
5308
5929
|
ids.push(r.id);
|
|
5309
5930
|
values.push(r.v);
|
|
5310
5931
|
columns.push(c === null || c === void 0 ? null : String(c));
|
|
5311
|
-
raws.push(misfit ?
|
|
5932
|
+
raws.push(misfit ? stored : null);
|
|
5312
5933
|
}
|
|
5313
5934
|
if (ids.length + fitIds.length > 0) {
|
|
5314
5935
|
const locked = [...ids, ...fitIds];
|
|
@@ -5620,7 +6241,7 @@ var PgDataset = class PgDataset {
|
|
|
5620
6241
|
if (!usesWalkGrammar(params)) try {
|
|
5621
6242
|
return await this.#read(type, params, false);
|
|
5622
6243
|
} catch (err) {
|
|
5623
|
-
throw this.#queryTimeout(type, err, params.signal) ?? err;
|
|
6244
|
+
throw this.#queryTimeout(type, err, params.signal) ?? storeBusy(err) ?? err;
|
|
5624
6245
|
}
|
|
5625
6246
|
if (!this.#inTx) return this.withTransaction((tx) => tx.read(type, params));
|
|
5626
6247
|
const ms = String(this.#statementTimeoutMs);
|
|
@@ -5631,7 +6252,7 @@ var PgDataset = class PgDataset {
|
|
|
5631
6252
|
WHERE name = 'statement_timeout' AND (setting::bigint = 0 OR setting::bigint > ${ms}::bigint)`, params.signal)).length > 0;
|
|
5632
6253
|
return await this.#read(type, params, true);
|
|
5633
6254
|
} catch (err) {
|
|
5634
|
-
throw this.#queryTimeout(type, err, params.signal, lowered ? this.#statementTimeoutMs : void 0) ?? err;
|
|
6255
|
+
throw this.#queryTimeout(type, err, params.signal, lowered ? this.#statementTimeoutMs : void 0) ?? storeBusy(err) ?? err;
|
|
5635
6256
|
}
|
|
5636
6257
|
}
|
|
5637
6258
|
/**
|
|
@@ -5707,10 +6328,15 @@ var PgDataset = class PgDataset {
|
|
|
5707
6328
|
}));
|
|
5708
6329
|
/** `statement`'s rows, but those of a stale read, which throw {@link StaleRead}; `key` is non-null on a real row. */
|
|
5709
6330
|
const run = async (statement, key) => {
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
5713
|
-
|
|
6331
|
+
try {
|
|
6332
|
+
if (guard === null) return await cancellable(statement, params.signal);
|
|
6333
|
+
const out = await cancellable(guardedRead(sql, guard, statement), params.signal);
|
|
6334
|
+
if (out[0]?.guard_ok !== true) throw new StaleRead(types, String(out[0]?.guard_iso));
|
|
6335
|
+
return out.filter((r) => r[key] !== null);
|
|
6336
|
+
} catch (err) {
|
|
6337
|
+
if (err?.code === "22003") throw new ResolverFrameError(VALUE_OUT_OF_RANGE, err.message);
|
|
6338
|
+
throw err;
|
|
6339
|
+
}
|
|
5714
6340
|
};
|
|
5715
6341
|
const clauses = [];
|
|
5716
6342
|
if (hasIdFilter) clauses.push(sql`r.id = ANY(${idList})`);
|
|
@@ -5761,55 +6387,94 @@ var PgDataset = class PgDataset {
|
|
|
5761
6387
|
};
|
|
5762
6388
|
}
|
|
5763
6389
|
const aggregate = params.aggregate ?? {};
|
|
5764
|
-
const
|
|
5765
|
-
|
|
5766
|
-
|
|
6390
|
+
const grouped = buildGroupedRead(sql, {
|
|
6391
|
+
...params,
|
|
6392
|
+
aggregate,
|
|
6393
|
+
limit
|
|
6394
|
+
}, matched, pickJoins, table, (f) => fieldDef(type, f));
|
|
6395
|
+
const rows = await run(sql`${withCtes(sql, scope)}${grouped.statement}`, "g");
|
|
6396
|
+
const truncated = limit !== void 0 && rows.length > limit;
|
|
5767
6397
|
return {
|
|
5768
6398
|
rows: [],
|
|
5769
6399
|
hasMore: false,
|
|
5770
|
-
groups: truncated ?
|
|
6400
|
+
groups: (truncated ? rows.slice(0, limit) : rows).map(grouped.decode),
|
|
5771
6401
|
...truncated ? { truncated: true } : {}
|
|
5772
6402
|
};
|
|
5773
6403
|
}
|
|
5774
6404
|
const orderBy = normalizeOrderBy(params.orderBy);
|
|
5775
6405
|
const orderByCastFor = bodyCastFor;
|
|
5776
6406
|
const joinShortIds = whereReferencesField(params.where, "shortId") || orderBy.some(([f]) => f === "shortId");
|
|
5777
|
-
const
|
|
5778
|
-
const
|
|
5779
|
-
|
|
5780
|
-
|
|
5781
|
-
const
|
|
5782
|
-
|
|
6407
|
+
const selectEntries = Object.entries(params.select ?? {});
|
|
6408
|
+
const selectFragments = /* @__PURE__ */ new Map();
|
|
6409
|
+
const selectTypes = /* @__PURE__ */ new Map();
|
|
6410
|
+
if (selectEntries.length > 0) {
|
|
6411
|
+
const used = JSON.stringify(params.select);
|
|
6412
|
+
const outputs = /* @__PURE__ */ new Map();
|
|
6413
|
+
const aggTypes = /* @__PURE__ */ new Map();
|
|
6414
|
+
for (const [name, spec] of Object.entries(frameAggs ?? {})) {
|
|
6415
|
+
const t = walkAggExprType(spec, type, fieldDef);
|
|
6416
|
+
if (!used.includes(`"_agg.${name}"`) || t === void 0 || scope?.aggValue === void 0) continue;
|
|
6417
|
+
const v = scope.aggValue(name);
|
|
6418
|
+
outputs.set(`_agg.${name}`, t === "number" ? sql`substrate_num(${v})::float8` : t === "date" ? sql`substrate_ts(${v})` : sql`(${v}) #>> '{}'`);
|
|
6419
|
+
aggTypes.set(`_agg.${name}`, t);
|
|
6420
|
+
}
|
|
6421
|
+
const fieldType = (name) => aggTypes.get(name) ?? exprTypeOfField(fieldDef(type, name));
|
|
6422
|
+
for (const [name, expr] of selectEntries) {
|
|
6423
|
+
const { fragment, type: t } = compileExprTyped(sql, expr, {
|
|
6424
|
+
table,
|
|
6425
|
+
alias: "r",
|
|
6426
|
+
outputs,
|
|
6427
|
+
fieldType
|
|
6428
|
+
});
|
|
6429
|
+
selectFragments.set(name, fragment);
|
|
6430
|
+
selectTypes.set(name, t ?? "string");
|
|
6431
|
+
}
|
|
6432
|
+
}
|
|
6433
|
+
const computedExpr = (field) => {
|
|
6434
|
+
const chosen = Object.hasOwn(params.select ?? {}, field) ? selectFragments.get(field) : void 0;
|
|
6435
|
+
if (chosen !== void 0) return selectTypes.get(field) === "string" ? sql`((${chosen}) COLLATE "C")` : chosen;
|
|
5783
6436
|
};
|
|
5784
6437
|
const lead = orderBy[0]?.[0] ?? "id";
|
|
5785
|
-
const fenced = joinShortIds ? void 0 :
|
|
5786
|
-
const
|
|
6438
|
+
const fenced = joinShortIds ? void 0 : computedExpr(lead);
|
|
6439
|
+
const computedFor = Object.assign((field) => fenced !== void 0 && field === lead ? sql`r.__computed_0` : computedExpr(field), { typeOf: (field) => selectTypes.get(field) });
|
|
5787
6440
|
const rel = sql.unsafe(table.from);
|
|
5788
|
-
const from = fenced === void 0 ? sql`${rel} r${pickJoins}${windowJoins}` : sql`(SELECT r.*, ${fenced} AS
|
|
6441
|
+
const from = fenced === void 0 ? sql`${rel} r${pickJoins}${windowJoins}` : sql`(SELECT r.*, ${fenced} AS __computed_0 FROM ${rel} r${pickJoins} WHERE ${combineAnd(sql, clauses)} OFFSET 0) r${windowJoins}`;
|
|
5789
6442
|
const ctes = withCtes(sql, scope);
|
|
5790
6443
|
const base = fenced === void 0 ? clauses : [];
|
|
5791
6444
|
let whereSql = combineAnd(sql, base);
|
|
5792
6445
|
let tailSql = null;
|
|
5793
6446
|
if (params.cursor !== void 0 && params.cursor !== "") {
|
|
5794
6447
|
const cursor = decodeCursor(params.cursor, orderBy);
|
|
5795
|
-
const cursorFragment = buildCursorCompare(sql, orderBy, cursor.orderByValues, cursor.id, orderByCastFor, table,
|
|
5796
|
-
const tail = buildCursorNullTail(sql, orderBy, cursor.orderByValues, orderByCastFor, table,
|
|
6448
|
+
const cursorFragment = buildCursorCompare(sql, orderBy, cursor.orderByValues, cursor.id, orderByCastFor, table, computedFor);
|
|
6449
|
+
const tail = buildCursorNullTail(sql, orderBy, cursor.orderByValues, orderByCastFor, table, computedFor);
|
|
5797
6450
|
if (tail !== null) tailSql = combineAnd(sql, [...base, tail]);
|
|
5798
6451
|
if (cursorFragment !== null) whereSql = combineAnd(sql, [...base, cursorFragment]);
|
|
5799
6452
|
}
|
|
5800
|
-
const orderBySql = buildOrderBy(sql, orderBy, orderByCastFor, table,
|
|
6453
|
+
const orderBySql = buildOrderBy(sql, orderBy, orderByCastFor, table, computedFor, "r");
|
|
5801
6454
|
const fetchLimit = limit !== void 0 ? limit + 1 : void 0;
|
|
5802
|
-
const
|
|
5803
|
-
const rowProjection = buildRowProjection(sql, table, params.fields, "r",
|
|
5804
|
-
const { fragment: sortKeyProjection, aliases: sortKeyAliases } = buildSortKeyProjection(sql, orderBy, bodyCastFor, table,
|
|
6455
|
+
const walk = scope === void 0 ? null : buildWalkColumns(sql, frameAggs, params.nested ?? {}, table, scope);
|
|
6456
|
+
const rowProjection = buildRowProjection(sql, table, params.fields, "r", walk?.select ?? null);
|
|
6457
|
+
const { fragment: sortKeyProjection, aliases: sortKeyAliases } = buildSortKeyProjection(sql, orderBy, bodyCastFor, table, computedFor);
|
|
5805
6458
|
const owner = sql.unsafe(`${sqlStringLiteral(this.namespace)}::text AS namespace, ${sqlStringLiteral(type)}::text AS type`);
|
|
5806
6459
|
const ownerOn = sql.unsafe(`s.namespace = ${sqlStringLiteral(this.namespace)} AND s.type = ${sqlStringLiteral(type)}`);
|
|
5807
6460
|
const inlineLimit = (n) => n !== void 0 ? sql.unsafe(`LIMIT ${BigInt(n)}`) : sql``;
|
|
5808
|
-
const
|
|
5809
|
-
|
|
5810
|
-
|
|
6461
|
+
const selectColumns = selectEntries.map(([name], i) => {
|
|
6462
|
+
const fragment = fenced !== void 0 && name === lead ? sql`r.__computed_0` : selectFragments.get(name);
|
|
6463
|
+
const alias = sql(`__sel_${i}`);
|
|
6464
|
+
switch (selectTypes.get(name)) {
|
|
6465
|
+
case "date": return sql`, to_char((${fragment}) AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS ${alias}`;
|
|
6466
|
+
case "int64": return sql`, (${fragment})::text AS ${alias}`;
|
|
6467
|
+
default: return sql`, ${fragment} AS ${alias}`;
|
|
6468
|
+
}
|
|
6469
|
+
});
|
|
6470
|
+
const selectProjection = selectColumns.length === 0 ? sql`` : selectColumns;
|
|
6471
|
+
const withMeta = readsRowMetadata(params.fields);
|
|
6472
|
+
const withShortId = readsShortId(params.fields);
|
|
6473
|
+
const metaColumns = withMeta ? sql`, r.created_at, r.updated_at,
|
|
5811
6474
|
r.created_by_kind, r.created_by_id,
|
|
5812
|
-
r.updated_by_kind, r.updated_by_id, r.seq
|
|
6475
|
+
r.updated_by_kind, r.updated_by_id, r.seq` : sql``;
|
|
6476
|
+
const statement = (where, n) => joinShortIds ? sql`
|
|
6477
|
+
SELECT r.id, ${rowProjection}, r.deleted_at, s.short_id${metaColumns}${selectProjection}${sortKeyProjection}
|
|
5813
6478
|
FROM ${from}
|
|
5814
6479
|
LEFT JOIN short_ids s
|
|
5815
6480
|
ON ${ownerOn}
|
|
@@ -5818,24 +6483,21 @@ var PgDataset = class PgDataset {
|
|
|
5818
6483
|
${orderBySql}
|
|
5819
6484
|
${n !== void 0 ? sql`LIMIT ${n}` : sql``}
|
|
5820
6485
|
` : sql`
|
|
5821
|
-
SELECT p
|
|
6486
|
+
SELECT p.*${withShortId ? sql`, s.short_id` : sql``}
|
|
5822
6487
|
FROM (
|
|
5823
|
-
SELECT r.id, ${rowProjection}, r.deleted_at,
|
|
5824
|
-
|
|
5825
|
-
r.created_by_kind, r.created_by_id,
|
|
5826
|
-
r.updated_by_kind, r.updated_by_id, r.seq,
|
|
5827
|
-
${owner}${sortKeyProjection}
|
|
6488
|
+
SELECT r.id, ${rowProjection}, r.deleted_at${metaColumns},
|
|
6489
|
+
${owner}${selectProjection}${sortKeyProjection}
|
|
5828
6490
|
FROM ${from}
|
|
5829
6491
|
WHERE ${where}
|
|
5830
6492
|
${orderBySql}
|
|
5831
6493
|
${inlineLimit(n)}
|
|
5832
6494
|
) p
|
|
5833
|
-
LEFT JOIN LATERAL (
|
|
6495
|
+
${withShortId ? sql`LEFT JOIN LATERAL (
|
|
5834
6496
|
SELECT s.short_id FROM short_ids s
|
|
5835
6497
|
WHERE s.namespace = p.namespace AND s.type = p.type AND s.row_id = p.id
|
|
5836
6498
|
LIMIT 1
|
|
5837
|
-
) s ON true
|
|
5838
|
-
${buildProjectedOrderBy(sql, orderBy, sortKeyAliases, orderByCastFor, table, "p")}
|
|
6499
|
+
) s ON true` : sql``}
|
|
6500
|
+
${buildProjectedOrderBy(sql, orderBy, sortKeyAliases, orderByCastFor, table, "p", computedFor)}
|
|
5839
6501
|
`;
|
|
5840
6502
|
const page = (where, n) => scope?.ctes === void 0 || scope.ctes.length === 0 ? statement(where, n) : sql`${ctes}${statement(where, n)}`;
|
|
5841
6503
|
let rows = await run(page(whereSql, fetchLimit), "id");
|
|
@@ -5845,7 +6507,18 @@ var PgDataset = class PgDataset {
|
|
|
5845
6507
|
rows = rows.concat(fetchLimit === void 0 ? tail : tail.slice(0, fetchLimit - rows.length));
|
|
5846
6508
|
}
|
|
5847
6509
|
const fields = this.#typeDefLookup?.(type, this.namespace)?.fields;
|
|
5848
|
-
const
|
|
6510
|
+
const orders = /* @__PURE__ */ new Map();
|
|
6511
|
+
const materialised = new Array(rows.length);
|
|
6512
|
+
for (let i = 0; i < rows.length; i++) {
|
|
6513
|
+
if (i > 0 && i % MATERIALIZE_BATCH_ROWS === 0) await new Promise((resolve) => setImmediate(resolve));
|
|
6514
|
+
const r = rows[i];
|
|
6515
|
+
const row = this.materialize(walk === null ? r : mergeWalkColumns(r, walk.columns, orders), fields, table, params.fields);
|
|
6516
|
+
for (const [k, [name]] of selectEntries.entries()) {
|
|
6517
|
+
const v = r[`__sel_${k}`];
|
|
6518
|
+
row[name] = v === null || v === void 0 ? null : selectTypes.get(name) === "int64" ? BigInt(v) : v;
|
|
6519
|
+
}
|
|
6520
|
+
materialised[i] = row;
|
|
6521
|
+
}
|
|
5849
6522
|
let hasMore = false;
|
|
5850
6523
|
let sliced = materialised;
|
|
5851
6524
|
if (limit !== void 0 && materialised.length > limit) {
|
|
@@ -6155,7 +6828,7 @@ var PgDataset = class PgDataset {
|
|
|
6155
6828
|
if (this.#inTx) return fn(this);
|
|
6156
6829
|
const rootSql = this.#sql;
|
|
6157
6830
|
const txTables = /* @__PURE__ */ new Map();
|
|
6158
|
-
const out = await rootSql.begin(async (txSql) => {
|
|
6831
|
+
const out = await busyAware(rootSql.begin(async (txSql) => {
|
|
6159
6832
|
const result = await fn(new PgDataset({
|
|
6160
6833
|
namespace: this.namespace,
|
|
6161
6834
|
sql: txSql,
|
|
@@ -6170,7 +6843,7 @@ var PgDataset = class PgDataset {
|
|
|
6170
6843
|
}));
|
|
6171
6844
|
await snapshotUnlessReadCommitted(txSql, txTables);
|
|
6172
6845
|
return result;
|
|
6173
|
-
});
|
|
6846
|
+
}));
|
|
6174
6847
|
this.#commitTables(txTables);
|
|
6175
6848
|
return out;
|
|
6176
6849
|
}
|
|
@@ -6205,11 +6878,12 @@ var PgDataset = class PgDataset {
|
|
|
6205
6878
|
};
|
|
6206
6879
|
};
|
|
6207
6880
|
if (this.#inTx) return fn(buildHandles(this.#sql));
|
|
6208
|
-
const
|
|
6881
|
+
const rootSql = this.#sql;
|
|
6882
|
+
const out = await busyAware(rootSql.begin(async (txSql) => {
|
|
6209
6883
|
const result = await fn(buildHandles(txSql));
|
|
6210
6884
|
await snapshotUnlessReadCommitted(txSql, txTables);
|
|
6211
6885
|
return result;
|
|
6212
|
-
});
|
|
6886
|
+
}));
|
|
6213
6887
|
this.#commitTables(txTables);
|
|
6214
6888
|
return out;
|
|
6215
6889
|
}
|
|
@@ -6552,6 +7226,17 @@ var PgDataset = class PgDataset {
|
|
|
6552
7226
|
}
|
|
6553
7227
|
});
|
|
6554
7228
|
}
|
|
7229
|
+
/** See `Dataset.poolStats`. Read on the root dataset; outside a transaction `#sql` is the pool. */
|
|
7230
|
+
poolStats() {
|
|
7231
|
+
const { connections, opening, idle, leased, waiting } = this.#sql.stats;
|
|
7232
|
+
return {
|
|
7233
|
+
connections,
|
|
7234
|
+
opening,
|
|
7235
|
+
idle,
|
|
7236
|
+
leased,
|
|
7237
|
+
waiting
|
|
7238
|
+
};
|
|
7239
|
+
}
|
|
6555
7240
|
/**
|
|
6556
7241
|
* See `Dataset.fieldWarnings`. Store-wide; none before the store is migrated. Counts
|
|
6557
7242
|
* only, since `/health` shows it unauthenticated: the named lines are in
|
|
@@ -6871,28 +7556,32 @@ var PgDataset = class PgDataset {
|
|
|
6871
7556
|
materialize(record, fields, table, requested) {
|
|
6872
7557
|
const body = decodeRow(table, record);
|
|
6873
7558
|
const raw = record.extra?.[RAW_KEY];
|
|
6874
|
-
|
|
6875
|
-
|
|
6876
|
-
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
if (k ===
|
|
6880
|
-
if (k === RESERVED_DELETED_AT) continue;
|
|
6881
|
-
if (k === "_metadata") continue;
|
|
6882
|
-
if (v === null && fields !== void 0 && Object.hasOwn(fields, k)) {
|
|
7559
|
+
const trimRaw = requested !== void 0 && raw !== null && typeof raw === "object" ? raw : void 0;
|
|
7560
|
+
const id = this.extractOriginalId(record.id, body);
|
|
7561
|
+
const kept = (k) => {
|
|
7562
|
+
if (k === "id" || k === RESERVED_DELETED_AT || k === "_metadata") return false;
|
|
7563
|
+
if (trimRaw !== void 0 && Object.hasOwn(trimRaw, k) && !Object.hasOwn(requested, k)) return false;
|
|
7564
|
+
if (body[k] === null && fields !== void 0 && Object.hasOwn(fields, k)) {
|
|
6883
7565
|
const def = fields[k];
|
|
6884
|
-
if (def !== void 0 && isColumnField(def))
|
|
7566
|
+
if (def !== void 0 && isColumnField(def)) return false;
|
|
6885
7567
|
}
|
|
6886
|
-
|
|
7568
|
+
return true;
|
|
7569
|
+
};
|
|
7570
|
+
const deletedAt = record.deleted_at !== null ? record.deleted_at : void 0;
|
|
7571
|
+
const shortId = record.short_id !== null && record.short_id !== void 0 ? Number(record.short_id) : void 0;
|
|
7572
|
+
const meta = readsRowMetadata(requested) ? assembleRowMetadata(record) : void 0;
|
|
7573
|
+
const special = (k) => k === RESERVED_DELETED_AT ? deletedAt : k === "shortId" ? shortId : k === "_metadata" ? meta : void 0;
|
|
7574
|
+
const out = requested === void 0 || !Object.hasOwn(requested, "id") ? { id } : {};
|
|
7575
|
+
if (requested !== void 0) {
|
|
7576
|
+
for (const k of Object.keys(requested)) if (k === "id") out.id = id;
|
|
7577
|
+
else if (special(k) !== void 0) out[k] = special(k);
|
|
7578
|
+
else if (Object.hasOwn(body, k) && kept(k)) out[k] = body[k];
|
|
6887
7579
|
}
|
|
6888
|
-
if (
|
|
6889
|
-
if (
|
|
6890
|
-
|
|
6891
|
-
if (meta !== void 0) out._metadata = meta;
|
|
6892
|
-
|
|
6893
|
-
const ordered = Object.hasOwn(requested, "id") ? {} : { id: out.id };
|
|
6894
|
-
for (const k of Object.keys(requested)) if (Object.hasOwn(out, k)) ordered[k] = out[k];
|
|
6895
|
-
return Object.assign(ordered, out);
|
|
7580
|
+
for (const k in body) if (!Object.hasOwn(out, k) && kept(k)) out[k] = special(k) ?? body[k];
|
|
7581
|
+
if (deletedAt !== void 0 && !Object.hasOwn(out, RESERVED_DELETED_AT)) out[RESERVED_DELETED_AT] = deletedAt;
|
|
7582
|
+
if (shortId !== void 0 && !Object.hasOwn(out, "shortId")) out.shortId = shortId;
|
|
7583
|
+
if (meta !== void 0 && !Object.hasOwn(out, "_metadata")) out._metadata = meta;
|
|
7584
|
+
return out;
|
|
6896
7585
|
}
|
|
6897
7586
|
extractOriginalId(textId, body) {
|
|
6898
7587
|
const bodyId = body.id;
|
|
@@ -6900,16 +7589,17 @@ var PgDataset = class PgDataset {
|
|
|
6900
7589
|
return textId;
|
|
6901
7590
|
}
|
|
6902
7591
|
};
|
|
7592
|
+
/** Rows materialised between event-loop yields on a read (a few ms of work). */
|
|
7593
|
+
const MATERIALIZE_BATCH_ROWS = 5e3;
|
|
6903
7594
|
/** The statement timeout a walk read runs under, in milliseconds, unless a shorter one is set. */
|
|
6904
7595
|
const WALK_STATEMENT_TIMEOUT_MS = 3e4;
|
|
6905
|
-
/** The
|
|
7596
|
+
/** The params that run under the statement timeout: `count: true`, and each non-empty object. */
|
|
6906
7597
|
function walkParams(params) {
|
|
6907
7598
|
const used = [
|
|
6908
7599
|
"aggregate",
|
|
6909
|
-
"bucket",
|
|
6910
|
-
"derive",
|
|
6911
7600
|
"agg",
|
|
6912
|
-
"nested"
|
|
7601
|
+
"nested",
|
|
7602
|
+
"select"
|
|
6913
7603
|
].filter((k) => {
|
|
6914
7604
|
const v = params[k];
|
|
6915
7605
|
return typeof v === "object" && v !== null && Object.keys(v).length > 0;
|
|
@@ -6920,22 +7610,98 @@ function walkParams(params) {
|
|
|
6920
7610
|
function usesWalkGrammar(params) {
|
|
6921
7611
|
return whereUsesWalks(params.where) || walkParams(params).length > 0;
|
|
6922
7612
|
}
|
|
7613
|
+
/**
|
|
7614
|
+
* The driver's `QUEUE_TIMEOUT` (no pooled connection came free within
|
|
7615
|
+
* `queue_timeout`) as `store_busy`, else null. The request never reached
|
|
7616
|
+
* Postgres, so a retry is safe.
|
|
7617
|
+
*/
|
|
7618
|
+
function storeBusy(err) {
|
|
7619
|
+
if (err?.code !== "QUEUE_TIMEOUT") return null;
|
|
7620
|
+
return new ResolverFrameError(STORE_BUSY, "the store is busy: no database connection came free in time; retry shortly");
|
|
7621
|
+
}
|
|
7622
|
+
/** Awaits `p`, rethrowing a `QUEUE_TIMEOUT` as `store_busy`. */
|
|
7623
|
+
async function busyAware(p) {
|
|
7624
|
+
try {
|
|
7625
|
+
return await p;
|
|
7626
|
+
} catch (err) {
|
|
7627
|
+
throw storeBusy(err) ?? err;
|
|
7628
|
+
}
|
|
7629
|
+
}
|
|
6923
7630
|
/** Awaits `query`, cancelling the statement in flight when `signal` aborts. */
|
|
6924
7631
|
async function cancellable(query, signal) {
|
|
6925
|
-
const cancel = () => {
|
|
6926
|
-
Promise.resolve(query.cancel()).catch(() => void 0);
|
|
6927
|
-
};
|
|
6928
|
-
signal?.addEventListener("abort", cancel, { once: true });
|
|
6929
7632
|
try {
|
|
6930
|
-
|
|
6931
|
-
return await query;
|
|
7633
|
+
return await query.signal(signal);
|
|
6932
7634
|
} catch (err) {
|
|
6933
7635
|
if (signal?.aborted === true) signal.throwIfAborted();
|
|
6934
7636
|
throw err;
|
|
6935
|
-
} finally {
|
|
6936
|
-
signal?.removeEventListener("abort", cancel);
|
|
6937
7637
|
}
|
|
6938
7638
|
}
|
|
6939
7639
|
|
|
6940
7640
|
//#endregion
|
|
6941
|
-
|
|
7641
|
+
//#region ../../packages/substrate-dataset-pg/src/pg-idempotency-store.ts
|
|
7642
|
+
/**
|
|
7643
|
+
* PgIdempotencyStore — PostgreSQL-backed `IdempotencyStore`. Spec: §5.13.
|
|
7644
|
+
*
|
|
7645
|
+
* Rows live in `idempotency` keyed by the engine's frame-scoped dedup
|
|
7646
|
+
* key; envelopes are stored as JSONB.
|
|
7647
|
+
*
|
|
7648
|
+
* Semantics:
|
|
7649
|
+
* - `get` returns `undefined` for unknown or expired keys. Expiry is lazy.
|
|
7650
|
+
* - `put` is first-write-wins for the envelope; a conflicting put only
|
|
7651
|
+
* extends `expires_at` (never shortens it).
|
|
7652
|
+
* - SQL errors propagate: a swallowed put failure would let the next retry
|
|
7653
|
+
* re-run the mutate.
|
|
7654
|
+
*/
|
|
7655
|
+
var PgIdempotencyStore = class {
|
|
7656
|
+
#sql;
|
|
7657
|
+
constructor(options) {
|
|
7658
|
+
this.#sql = options.sql;
|
|
7659
|
+
}
|
|
7660
|
+
async get(key) {
|
|
7661
|
+
const row = (await this.#sql`
|
|
7662
|
+
SELECT envelope
|
|
7663
|
+
FROM idempotency
|
|
7664
|
+
WHERE key = ${key}
|
|
7665
|
+
AND expires_at > now()
|
|
7666
|
+
LIMIT 1
|
|
7667
|
+
`)[0];
|
|
7668
|
+
if (row === void 0) return void 0;
|
|
7669
|
+
return row.envelope;
|
|
7670
|
+
}
|
|
7671
|
+
async put(key, envelope, ttlMs) {
|
|
7672
|
+
await this.#sql`
|
|
7673
|
+
INSERT INTO idempotency (key, envelope, expires_at)
|
|
7674
|
+
VALUES (
|
|
7675
|
+
${key},
|
|
7676
|
+
${stringifyJson(envelope)}::text::jsonb,
|
|
7677
|
+
now() + (${ttlMs}::bigint || ' milliseconds')::interval
|
|
7678
|
+
)
|
|
7679
|
+
ON CONFLICT (key) DO UPDATE SET
|
|
7680
|
+
-- The envelope is IMMUTABLE while the entry is live: an idempotency key
|
|
7681
|
+
-- must resolve to one stable answer for its whole window, or two
|
|
7682
|
+
-- retries of the same request can get different results, which is the
|
|
7683
|
+
-- thing the store exists to prevent.
|
|
7684
|
+
--
|
|
7685
|
+
-- Once it has EXPIRED that rule inverts. The dedupe window closed, the
|
|
7686
|
+
-- mutation legitimately re-executed, and the new envelope is the real
|
|
7687
|
+
-- result. Keeping the old one would hand back a stale answer wrapped in
|
|
7688
|
+
-- a fresh TTL, to a caller who was never deduped at all.
|
|
7689
|
+
envelope = CASE
|
|
7690
|
+
WHEN idempotency.expires_at <= now() THEN EXCLUDED.envelope
|
|
7691
|
+
ELSE idempotency.envelope
|
|
7692
|
+
END,
|
|
7693
|
+
-- GREATEST rather than a WHERE guard: extends a live window, never
|
|
7694
|
+
-- shortens it, and takes the new value outright when the old is past.
|
|
7695
|
+
expires_at = GREATEST(idempotency.expires_at, EXCLUDED.expires_at)
|
|
7696
|
+
`;
|
|
7697
|
+
}
|
|
7698
|
+
async reapExpired() {
|
|
7699
|
+
return { deleted: (await this.#sql`
|
|
7700
|
+
DELETE FROM idempotency
|
|
7701
|
+
WHERE expires_at <= now()
|
|
7702
|
+
`).count };
|
|
7703
|
+
}
|
|
7704
|
+
};
|
|
7705
|
+
|
|
7706
|
+
//#endregion
|
|
7707
|
+
export { copyRows as C, detectLayout as S, tryWithPgAdvisoryLock as T, pendingMigrations as _, ensureTypeTable as a, UUID_CHECK_MIGRATION as b, typeTableRef as c, PLAIN_SCHEMA_NAME as d, assertNotMigrating as f, isDerivedMigration as g, deriveListIndexMigration as h, seqName as i, whileNoTypeTable as l, assertTablesLayout as m, PgDataset as n, guardedRefRead as o, assertStoreSchema as p, txScopeIdFor as r, loadTypeTable as s, PgIdempotencyStore as t, MIGRATE_STORE_MARKER_TABLE as u, runDerivedMigrations as v, parseSourceBody as w, UUID_REGEX as x, runMigrations as y };
|