@sixb/pg 0.2.1 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/agents/computers.d.ts +8 -2
- package/dist/agents/computers.d.ts.map +1 -1
- package/dist/auth-storage/auth-storage.d.ts +2 -1
- package/dist/auth-storage/auth-storage.d.ts.map +1 -1
- package/dist/auth-storage/identities.d.ts +4 -0
- package/dist/auth-storage/identities.d.ts.map +1 -1
- package/dist/auth-storage/rows.d.ts +2 -1
- package/dist/auth-storage/rows.d.ts.map +1 -1
- package/dist/auth-storage/users.d.ts.map +1 -1
- package/dist/connector-connection-storage.d.ts.map +1 -1
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2377 -428
- package/dist/index.js.map +37 -30
- package/dist/migrations.d.ts.map +1 -1
- package/dist/objects/conditional-reader.d.ts +7 -2
- package/dist/objects/conditional-reader.d.ts.map +1 -1
- package/dist/objects/partitions.d.ts.map +1 -1
- package/dist/objects/prepare-queries.d.ts.map +1 -1
- package/dist/objects/query-compiler.d.ts +14 -1
- package/dist/objects/query-compiler.d.ts.map +1 -1
- package/dist/objects/query-indexes.d.ts +13 -1
- package/dist/objects/query-indexes.d.ts.map +1 -1
- package/dist/objects/read-scope.d.ts.map +1 -1
- package/dist/objects/reader.d.ts +14 -1
- package/dist/objects/reader.d.ts.map +1 -1
- package/dist/objects/storage.d.ts +11 -1
- package/dist/objects/storage.d.ts.map +1 -1
- package/dist/objects/visibility-indexes.d.ts +22 -0
- package/dist/objects/visibility-indexes.d.ts.map +1 -0
- package/dist/objects/visibility-statistics.d.ts +26 -0
- package/dist/objects/visibility-statistics.d.ts.map +1 -0
- package/dist/objects/visibility.d.ts +34 -0
- package/dist/objects/visibility.d.ts.map +1 -0
- package/dist/ontology-storage/materialization-session.d.ts +5 -0
- package/dist/ontology-storage/materialization-session.d.ts.map +1 -1
- package/dist/ontology-storage/materialization-writer.d.ts +14 -0
- package/dist/ontology-storage/materialization-writer.d.ts.map +1 -1
- package/dist/ontology-storage/materializations.d.ts.map +1 -1
- package/dist/pg-artifact-storage.d.ts +14 -2
- package/dist/pg-artifact-storage.d.ts.map +1 -1
- package/dist/pg-client.d.ts.map +1 -1
- package/dist/pg-dataset-ingest-storage.d.ts +6 -0
- package/dist/pg-dataset-ingest-storage.d.ts.map +1 -0
- package/dist/pg-lake-retention-run-storage.d.ts +11 -0
- package/dist/pg-lake-retention-run-storage.d.ts.map +1 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts +11 -0
- package/dist/pg-maintenance-heartbeat-storage.d.ts.map +1 -0
- package/dist/pg-pipeline-run-storage.d.ts +5 -2
- package/dist/pg-pipeline-run-storage.d.ts.map +1 -1
- package/dist/pg-projection-run-storage.d.ts +5 -1
- package/dist/pg-projection-run-storage.d.ts.map +1 -1
- package/dist/pg-sync-run-storage.d.ts +6 -2
- package/dist/pg-sync-run-storage.d.ts.map +1 -1
- package/dist/pg-webhook-run-storage.d.ts +2 -1
- package/dist/pg-webhook-run-storage.d.ts.map +1 -1
- package/dist/pg-workflow-run-storage.d.ts +4 -3
- package/dist/pg-workflow-run-storage.d.ts.map +1 -1
- package/dist/run-claim-sql.d.ts +37 -0
- package/dist/run-claim-sql.d.ts.map +1 -0
- package/dist/share-session-storage.d.ts +2 -1
- package/dist/share-session-storage.d.ts.map +1 -1
- package/package.json +3 -3
- package/src/agents/computers.ts +54 -5
- package/src/auth-storage/auth-storage.ts +66 -0
- package/src/auth-storage/identities.ts +15 -0
- package/src/auth-storage/rows.ts +3 -0
- package/src/auth-storage/users.ts +3 -0
- package/src/connector-connection-storage.ts +33 -16
- package/src/index.ts +15 -0
- package/src/migrations/087-agent-computer-background-commands.sql +4 -0
- package/src/migrations/088-web-search-model-kind.sql +12 -0
- package/src/migrations/089-maintenance-heartbeats.sql +10 -0
- package/src/migrations/090-lake-retention-runs.sql +8 -0
- package/src/migrations/091-dataset-ingest-inbox.sql +38 -0
- package/src/migrations/092-run-recovery.sql +23 -0
- package/src/migrations/093-webhook-and-projection-leases.sql +10 -0
- package/src/migrations/094-artifact-board-previews.sql +14 -0
- package/src/migrations/095-artifact-open-versions.sql +101 -0
- package/src/migrations/096-artifact-documents.sql +11 -0
- package/src/migrations/097-artifact-frame-files.sql +8 -0
- package/src/migrations/098-artifact-comments.sql +36 -0
- package/src/migrations/099-object-artifact-refs.sql +12 -0
- package/src/migrations/100-shared-link-guests.sql +17 -0
- package/src/migrations/101-connector-connection-owners.sql +38 -0
- package/src/migrations/102-user-suspension-source.sql +12 -0
- package/src/migrations.ts +46 -0
- package/src/objects/conditional-reader.ts +26 -4
- package/src/objects/partitions.ts +17 -4
- package/src/objects/prepare-queries.ts +8 -3
- package/src/objects/query-compiler.ts +64 -23
- package/src/objects/query-indexes.ts +93 -27
- package/src/objects/read-scope.ts +147 -77
- package/src/objects/reader.ts +46 -18
- package/src/objects/storage.ts +53 -0
- package/src/objects/visibility-indexes.ts +222 -0
- package/src/objects/visibility-statistics.ts +73 -0
- package/src/objects/visibility.ts +135 -0
- package/src/ontology-storage/materialization-session.ts +4 -0
- package/src/ontology-storage/materialization-writer.ts +68 -1
- package/src/ontology-storage/materializations.ts +6 -2
- package/src/pg-artifact-storage.ts +798 -137
- package/src/pg-client.ts +4 -0
- package/src/pg-dataset-ingest-storage.ts +305 -0
- package/src/pg-lake-retention-run-storage.ts +40 -0
- package/src/pg-maintenance-heartbeat-storage.ts +53 -0
- package/src/pg-pipeline-run-storage.ts +85 -42
- package/src/pg-projection-run-storage.ts +61 -2
- package/src/pg-sync-run-storage.ts +100 -37
- package/src/pg-webhook-run-storage.ts +37 -11
- package/src/pg-workflow-run-storage.ts +39 -100
- package/src/run-claim-sql.ts +173 -0
- package/src/share-session-storage.ts +20 -0
|
@@ -13,8 +13,10 @@ import {
|
|
|
13
13
|
isVectorProfileQuery,
|
|
14
14
|
linkEndpointObjectTypeIds,
|
|
15
15
|
normalizeVector,
|
|
16
|
+
sortTieDirection,
|
|
16
17
|
vectorSearchCandidateLimit,
|
|
17
18
|
} from "@sixb/core/internal/query"
|
|
19
|
+
import type { PgSortIndexRead } from "./query-indexes"
|
|
18
20
|
|
|
19
21
|
export interface PgObjectQueryPageRow {
|
|
20
22
|
object_type_id: string
|
|
@@ -117,7 +119,12 @@ export function compilePgObjectQuery(
|
|
|
117
119
|
includeTotal?: boolean
|
|
118
120
|
source?: PgObjectQuerySource
|
|
119
121
|
correlatedTraversal?: boolean
|
|
120
|
-
|
|
122
|
+
sortIndex?: PgSortIndexRead
|
|
123
|
+
/**
|
|
124
|
+
* Read the objects the query keeps before ordering them, never along an index that orders
|
|
125
|
+
* them: for a reader who sees few objects of the type, a walk would check nearly all of them.
|
|
126
|
+
*/
|
|
127
|
+
unorderedStart?: boolean
|
|
121
128
|
} = {}
|
|
122
129
|
): CompiledPgObjectQuery {
|
|
123
130
|
if (hasVectorProfile(query) && !isVectorProfileQuery(query)) {
|
|
@@ -131,10 +138,12 @@ export function compilePgObjectQuery(
|
|
|
131
138
|
compileObjectQueryInternal(projectId, query, {
|
|
132
139
|
probeLimit: options.includeTotal === false,
|
|
133
140
|
correlatedTraversal: options.correlatedTraversal,
|
|
141
|
+
// The fence that isolates a sort index's overflow rows keeps any index order out of them.
|
|
142
|
+
isolateSortOverflow: options.unorderedStart,
|
|
134
143
|
source,
|
|
135
144
|
})
|
|
136
145
|
)
|
|
137
|
-
if (options.
|
|
146
|
+
if (options.sortIndex) {
|
|
138
147
|
let bounded = query
|
|
139
148
|
const expansions: (readonly ObjectExpansion[])[] = []
|
|
140
149
|
while (bounded.kind === "expand") {
|
|
@@ -142,25 +151,33 @@ export function compilePgObjectQuery(
|
|
|
142
151
|
bounded = bounded.input
|
|
143
152
|
}
|
|
144
153
|
if (bounded.kind === "page" || bounded.kind === "limit") {
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
+
const { bound, nullRank } = options.sortIndex
|
|
155
|
+
|
|
156
|
+
// Walked backwards, the index holds the null values first: they are read as a range of
|
|
157
|
+
// their own, so each range is one walk.
|
|
158
|
+
const indexed = nullRank
|
|
159
|
+
? [`${bound} AND ${nullRank} = 0`, `${bound} AND ${nullRank} = 1`]
|
|
160
|
+
: [bound]
|
|
161
|
+
|
|
162
|
+
const branches = [...indexed, `NOT (${bound})`].map((startPredicate, branch) =>
|
|
163
|
+
compileObjectQueryInternal(projectId, bounded, {
|
|
164
|
+
probeLimit: options.includeTotal === false,
|
|
165
|
+
source,
|
|
166
|
+
startPredicate,
|
|
167
|
+
isolateSortOverflow: branch === indexed.length,
|
|
168
|
+
correlatedTraversal: options.correlatedTraversal,
|
|
169
|
+
})
|
|
154
170
|
)
|
|
155
171
|
const limit =
|
|
156
172
|
bounded.kind === "page"
|
|
157
173
|
? bounded.pageSize + 1
|
|
158
174
|
: bounded.limit + (options.includeTotal === false ? 1 : 0)
|
|
175
|
+
|
|
159
176
|
// Links are expanded once, for the page the branches merge into: each further reference
|
|
160
177
|
// to objects costs the planner a few milliseconds over its prepared indexes.
|
|
161
178
|
let rows: CompiledPredicate = {
|
|
162
|
-
sql: `SELECT * FROM (
|
|
163
|
-
args: [...branches
|
|
179
|
+
sql: `SELECT * FROM (${branches.map((branch) => `(${branch.sql})`).join(" UNION ALL ")}) AS input ORDER BY ${compiled.order.sql} LIMIT ?`,
|
|
180
|
+
args: [...branches.flatMap((branch) => branch.args), ...compiled.order.args, limit],
|
|
164
181
|
}
|
|
165
182
|
for (const expansion of expansions) {
|
|
166
183
|
rows = expandRows({ ...rows, order: compiled.order }, expansion, {
|
|
@@ -168,7 +185,8 @@ export function compilePgObjectQuery(
|
|
|
168
185
|
source,
|
|
169
186
|
})
|
|
170
187
|
}
|
|
171
|
-
|
|
188
|
+
// The rows replace the wrapped statement, so they take the source's relations again.
|
|
189
|
+
compiled = { ...compiled, ...source.wrapStatement(rows.sql, rows.args) }
|
|
172
190
|
}
|
|
173
191
|
}
|
|
174
192
|
const traversal = !options.correlatedTraversal && boundedTraversal(query)
|
|
@@ -194,6 +212,13 @@ export function compilePgObjectQuery(
|
|
|
194
212
|
export interface PgObjectQuerySource {
|
|
195
213
|
readonly objectsTable: string
|
|
196
214
|
readonly linksTable: string
|
|
215
|
+
/**
|
|
216
|
+
* The links a statement reads where it already joins both endpoints through `objectsTable`, or
|
|
217
|
+
* only sizes a traversal, when a link's visibility and contents follow from its endpoints alone.
|
|
218
|
+
* Unset, it reads `linksTable`: a selected scope reaches links by path and redacts their
|
|
219
|
+
* properties.
|
|
220
|
+
*/
|
|
221
|
+
readonly linksBetweenVisibleObjectsTable?: string
|
|
197
222
|
readonly objectPropertyPermissionsTable?: string
|
|
198
223
|
/**
|
|
199
224
|
* Where to read one concrete type's objects, when the source can say it more precisely than
|
|
@@ -217,6 +242,11 @@ export interface PgTypedObjects {
|
|
|
217
242
|
readonly filter?: { readonly sql: string; readonly args: readonly unknown[] }
|
|
218
243
|
}
|
|
219
244
|
|
|
245
|
+
/** The links a statement reads once it joins both of their endpoints through `objectsTable`. */
|
|
246
|
+
function linksBetweenVisibleObjects(source: PgObjectQuerySource): string {
|
|
247
|
+
return source.linksBetweenVisibleObjectsTable ?? source.linksTable
|
|
248
|
+
}
|
|
249
|
+
|
|
220
250
|
/** The table and `AND ...` filter a statement reads one type's objects with. */
|
|
221
251
|
export function pgObjectsOfType(
|
|
222
252
|
source: PgObjectQuerySource,
|
|
@@ -834,6 +864,9 @@ function compileTraversalRelation(
|
|
|
834
864
|
const parent = incoming ? "target" : "source"
|
|
835
865
|
const endpoint = incoming ? "source" : "target"
|
|
836
866
|
const endpointType = compileEndpointTypes("selected", linkEndpointObjectTypeIds(traversal))
|
|
867
|
+
|
|
868
|
+
// The input rows were read through the source and `selected` is too, so both ends of every
|
|
869
|
+
// edge read here are visible.
|
|
837
870
|
return {
|
|
838
871
|
sql: `FROM ${source.objectsTable} AS selected
|
|
839
872
|
WHERE selected.project_id = ?
|
|
@@ -841,7 +874,7 @@ function compileTraversalRelation(
|
|
|
841
874
|
${endpointPredicate ? `AND (${endpointPredicate})` : ""}
|
|
842
875
|
AND EXISTS (
|
|
843
876
|
SELECT 1 FROM (${input.sql}) AS input
|
|
844
|
-
JOIN ${source
|
|
877
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge
|
|
845
878
|
ON edge.project_id = input.project_id
|
|
846
879
|
AND edge.${parent}_type_id = input.object_type_id
|
|
847
880
|
AND edge.${parent}_id = input.primary_id
|
|
@@ -900,9 +933,12 @@ function compileTraversalProbe(
|
|
|
900
933
|
const input = compileAggregateSource(projectId, traversal.input, source)
|
|
901
934
|
const parent = traversal.direction === "incoming" ? "target" : "source"
|
|
902
935
|
const limit = 1000
|
|
936
|
+
|
|
937
|
+
// The probe only sizes the traversal and returns no row, so it counts every edge the traversal
|
|
938
|
+
// walks from visible inputs, including those its far-end join will then drop.
|
|
903
939
|
const statement = compilePgObjectStatement(
|
|
904
940
|
`SELECT 1 FROM (${input.sql}) AS input
|
|
905
|
-
JOIN ${source
|
|
941
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
|
|
906
942
|
AND edge.${parent}_type_id = input.object_type_id AND edge.${parent}_id = input.primary_id
|
|
907
943
|
WHERE edge.link_id = ? ${traversal.direction === "incoming" && traversal.sourceObjectTypeId ? "AND edge.source_type_id = ?" : ""}
|
|
908
944
|
LIMIT ?`,
|
|
@@ -1037,8 +1073,10 @@ function compileExpansionValue(
|
|
|
1037
1073
|
whereArgs.push(expansion.sourceObjectTypeId)
|
|
1038
1074
|
}
|
|
1039
1075
|
|
|
1076
|
+
// The parent is a row read through the source and the neighbour is joined from its objects, so
|
|
1077
|
+
// both ends of every edge read here are visible.
|
|
1040
1078
|
const from = `
|
|
1041
|
-
FROM ${ctx.source
|
|
1079
|
+
FROM ${linksBetweenVisibleObjects(ctx.source)} AS ${edge}
|
|
1042
1080
|
JOIN ${ctx.source.objectsTable} AS ${target}
|
|
1043
1081
|
ON ${target}.project_id = ${edge}.project_id
|
|
1044
1082
|
AND ${target}.object_type_id = ${neighborType}
|
|
@@ -1134,7 +1172,8 @@ function compileExpansionOrder(
|
|
|
1134
1172
|
args.push(field.propertyId, field.propertyId, field.propertyId)
|
|
1135
1173
|
}
|
|
1136
1174
|
|
|
1137
|
-
|
|
1175
|
+
const ties = sortTieDirection(expansion.orderBy ?? []).toUpperCase()
|
|
1176
|
+
clauses.push(`${neighborType} ${ties}`, `${neighborId} ${ties}`)
|
|
1138
1177
|
return { sql: clauses.join(", "), args }
|
|
1139
1178
|
}
|
|
1140
1179
|
|
|
@@ -1365,6 +1404,8 @@ function compileAggregateTraversal(
|
|
|
1365
1404
|
// Deduplicate endpoint identities before fetching objects. Bounded arrays let PostgreSQL
|
|
1366
1405
|
// visit primary keys in groups instead of doing one random B-tree descent per edge.
|
|
1367
1406
|
// Each array contains at most 1024 IDs; larger traversals can spill their sort/group work.
|
|
1407
|
+
// Both ends of each edge are visible: the input was read through the source, and the sources
|
|
1408
|
+
// are joined from its objects.
|
|
1368
1409
|
return {
|
|
1369
1410
|
sql: `SELECT selected.project_id, selected.object_type_id, selected.primary_id, selected.properties
|
|
1370
1411
|
FROM (
|
|
@@ -1373,7 +1414,7 @@ function compileAggregateTraversal(
|
|
|
1373
1414
|
FROM (
|
|
1374
1415
|
SELECT DISTINCT edge.source_id AS primary_id
|
|
1375
1416
|
FROM (${input.sql}) AS input
|
|
1376
|
-
JOIN ${source
|
|
1417
|
+
JOIN ${linksBetweenVisibleObjects(source)} AS edge ON edge.project_id = input.project_id
|
|
1377
1418
|
AND edge.target_type_id = input.object_type_id AND edge.target_id = input.primary_id
|
|
1378
1419
|
WHERE edge.link_id = ? AND edge.source_type_id = ?
|
|
1379
1420
|
) AS endpoints
|
|
@@ -1834,14 +1875,14 @@ function sortOrderFields(fields: readonly ObjectQuerySortField[]): readonly Comp
|
|
|
1834
1875
|
scalarKind: field.scalarKind,
|
|
1835
1876
|
})
|
|
1836
1877
|
}
|
|
1837
|
-
orderFields.push(...identityOrderFields())
|
|
1878
|
+
orderFields.push(...identityOrderFields(sortTieDirection(fields)))
|
|
1838
1879
|
return orderFields
|
|
1839
1880
|
}
|
|
1840
1881
|
|
|
1841
|
-
function identityOrderFields(): readonly CompiledOrderField[] {
|
|
1882
|
+
function identityOrderFields(direction: "asc" | "desc" = "asc"): readonly CompiledOrderField[] {
|
|
1842
1883
|
return [
|
|
1843
|
-
{ kind: "column", column: "object_type_id", direction
|
|
1844
|
-
{ kind: "column", column: "primary_id", direction
|
|
1884
|
+
{ kind: "column", column: "object_type_id", direction },
|
|
1885
|
+
{ kind: "column", column: "primary_id", direction },
|
|
1845
1886
|
]
|
|
1846
1887
|
}
|
|
1847
1888
|
|
|
@@ -3,7 +3,7 @@ import type { ObjectQuery, QueryScalarKind } from "@sixb/core"
|
|
|
3
3
|
import { linkEndpointObjectTypeIds } from "@sixb/core/internal/query"
|
|
4
4
|
import type { ReservedSQL, SQL, SQLClient } from "../pg-client"
|
|
5
5
|
import { isConnectionLost } from "../storage-errors"
|
|
6
|
-
import { withReservedPgConnection } from "../transactions"
|
|
6
|
+
import { runPgTransactionOn, withReservedPgConnection } from "../transactions"
|
|
7
7
|
import { ensureObjectPartition, objectPartitionName } from "./partitions"
|
|
8
8
|
import { lockPgPreparation } from "./preparation-lock"
|
|
9
9
|
import { hasJsonValueIndex, isTextScalarKind } from "./query-compiler"
|
|
@@ -136,6 +136,11 @@ async function createQueryIndex(
|
|
|
136
136
|
return name
|
|
137
137
|
}
|
|
138
138
|
|
|
139
|
+
// PostgreSQL builds statistics on each expression whatever the kinds. Of the multivariate ones,
|
|
140
|
+
// the most-common-values list estimates every combination of conditions these cover; ndistinct and
|
|
141
|
+
// dependencies changed no estimate on the read benchmark and doubled the time ANALYZE took.
|
|
142
|
+
const STATISTICS_KINDS = "mcv"
|
|
143
|
+
|
|
139
144
|
async function ensureExpressionStatistics(
|
|
140
145
|
connection: ReservedSQL,
|
|
141
146
|
schema: string,
|
|
@@ -155,25 +160,48 @@ async function ensureExpressionStatistics(
|
|
|
155
160
|
|
|
156
161
|
for (const [objectTypeId, expressions] of expressionsByType) {
|
|
157
162
|
const partition = `${identifier(schema)}.${identifier(objectPartitionName(objectTypeId))}`
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
163
|
+
|
|
164
|
+
// One transaction, so queries plan with the old statistics until the new ones are built.
|
|
165
|
+
await runPgTransactionOn(connection, async () => {
|
|
166
|
+
let statisticsChanged = false
|
|
167
|
+
for (const expression of expressions) {
|
|
168
|
+
const name = statisticsName(objectTypeId, `${STATISTICS_KINDS}:${expression}`)
|
|
169
|
+
if (await statisticsExist(connection, schema, name)) continue
|
|
170
|
+
|
|
171
|
+
// Built with every kind before the kinds were part of the name.
|
|
172
|
+
const superseded = statisticsName(objectTypeId, expression)
|
|
173
|
+
if (await statisticsExist(connection, schema, superseded)) {
|
|
174
|
+
await connection.unsafe(`DROP STATISTICS ${identifier(schema)}.${identifier(superseded)}`)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
await connection.unsafe(
|
|
178
|
+
`CREATE STATISTICS ${identifier(schema)}.${identifier(name)} (${STATISTICS_KINDS}) ON project_id, ${expression} FROM ${partition}`
|
|
179
|
+
)
|
|
180
|
+
statisticsChanged = true
|
|
181
|
+
}
|
|
182
|
+
if (statisticsChanged) await connection.unsafe(`ANALYZE ${partition}`)
|
|
183
|
+
})
|
|
174
184
|
}
|
|
175
185
|
}
|
|
176
186
|
|
|
187
|
+
function statisticsName(objectTypeId: string, definition: string): string {
|
|
188
|
+
return `sixb_query_stats_${createHash("sha256")
|
|
189
|
+
.update(`${objectTypeId}:${definition}`)
|
|
190
|
+
.digest("hex")
|
|
191
|
+
.slice(0, 24)}`
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
async function statisticsExist(
|
|
195
|
+
connection: ReservedSQL,
|
|
196
|
+
schema: string,
|
|
197
|
+
name: string
|
|
198
|
+
): Promise<boolean> {
|
|
199
|
+
const [existing] = await connection`
|
|
200
|
+
SELECT 1 FROM pg_statistic_ext s JOIN pg_namespace n ON n.oid = s.stxnamespace
|
|
201
|
+
WHERE n.nspname = ${schema} AND s.stxname = ${name}`
|
|
202
|
+
return existing !== undefined
|
|
203
|
+
}
|
|
204
|
+
|
|
177
205
|
function statisticsExpressions(
|
|
178
206
|
index: Exclude<PgObjectQueryIndex, { kind: "incomingLinks" }>
|
|
179
207
|
): string[] {
|
|
@@ -244,17 +272,24 @@ function compileIndex(index: PgObjectQueryIndex): string {
|
|
|
244
272
|
fields.push(`(properties ->> ${literal(propertyId)})`)
|
|
245
273
|
}
|
|
246
274
|
for (const field of index.fields) {
|
|
247
|
-
const property = `properties -> ${literal(field.propertyId)}`
|
|
248
275
|
fields.push(
|
|
249
|
-
|
|
276
|
+
sortIndexNullRank(field.propertyId),
|
|
250
277
|
`${sortValue(field)} ${field.direction === "desc" ? "DESC" : "ASC"}`
|
|
251
278
|
)
|
|
252
279
|
}
|
|
253
|
-
|
|
280
|
+
|
|
281
|
+
// Ties follow the last field, as queries order them.
|
|
282
|
+
fields.push(index.fields.at(-1)?.direction === "desc" ? "primary_id DESC" : "primary_id")
|
|
254
283
|
}
|
|
255
284
|
return `(${fields.join(", ")}) WHERE ${predicate.join(" AND ")}`
|
|
256
285
|
}
|
|
257
286
|
|
|
287
|
+
/** 1 for a null or missing value, which sorts last in both directions, 0 otherwise. */
|
|
288
|
+
function sortIndexNullRank(propertyId: string): string {
|
|
289
|
+
const property = `properties -> ${literal(propertyId)}`
|
|
290
|
+
return `(CASE WHEN jsonb_typeof(${property}) IS NULL OR jsonb_typeof(${property}) = 'null' THEN 1 ELSE 0 END)`
|
|
291
|
+
}
|
|
292
|
+
|
|
258
293
|
/** The query compiler's ordered value expression; an index serves a sort only when they match. */
|
|
259
294
|
function sortValue(field: Extract<PgObjectQueryIndex, { kind: "sort" }>["fields"][number]): string {
|
|
260
295
|
const text = `(properties ->> ${literal(field.propertyId)})`
|
|
@@ -337,14 +372,27 @@ async function readPgSortIndexes(sql: SQLClient): Promise<readonly PgSortIndex[]
|
|
|
337
372
|
return indexes
|
|
338
373
|
}
|
|
339
374
|
|
|
375
|
+
/** How an ordered read walks a prepared sort index. */
|
|
376
|
+
export interface PgSortIndexRead {
|
|
377
|
+
/** The type whose objects the index orders. */
|
|
378
|
+
readonly objectTypeId: string
|
|
379
|
+
/** The rows the index holds; the others are read through its overflow index. */
|
|
380
|
+
readonly bound: string
|
|
381
|
+
/**
|
|
382
|
+
* Set when the read walks the index backwards: the field's null rank, whose two values are read
|
|
383
|
+
* apart, since null values sort last in both directions.
|
|
384
|
+
*/
|
|
385
|
+
readonly nullRank?: string
|
|
386
|
+
}
|
|
387
|
+
|
|
340
388
|
/** Only split a flat, bounded ordered read; joins, nested limits and projections keep the
|
|
341
389
|
* general compiler. Its unrestricted fallback always preserves semantics. */
|
|
342
|
-
export async function
|
|
390
|
+
export async function findPgSortIndexRead(
|
|
343
391
|
sql: SQLClient,
|
|
344
392
|
catalog: PgSortIndexCatalog,
|
|
345
393
|
projectId: string,
|
|
346
394
|
query: ObjectQuery
|
|
347
|
-
): Promise<
|
|
395
|
+
): Promise<PgSortIndexRead | undefined> {
|
|
348
396
|
while (query.kind === "expand") query = query.input
|
|
349
397
|
if (query.kind !== "page" && query.kind !== "limit") return undefined
|
|
350
398
|
let input = query.input
|
|
@@ -387,7 +435,7 @@ export async function findPgSortIndexBound(
|
|
|
387
435
|
if (!objectTypeId || !fields?.length || fields.some((field) => field.kind !== "property"))
|
|
388
436
|
return undefined
|
|
389
437
|
|
|
390
|
-
const matches: PgSortIndex[] = []
|
|
438
|
+
const matches: { index: PgSortIndex; backwards: boolean }[] = []
|
|
391
439
|
for (const index of await catalog.indexes(sql)) {
|
|
392
440
|
// Metadata is generated by this module. Verify all SQL-interpolated members before using it.
|
|
393
441
|
if (
|
|
@@ -411,14 +459,32 @@ export async function findPgSortIndexBound(
|
|
|
411
459
|
typeof field.propertyId === "string" &&
|
|
412
460
|
fields[i]?.kind === "property" &&
|
|
413
461
|
fields[i].propertyId === field.propertyId &&
|
|
414
|
-
(field.direction ?? "asc") === (fields[i].direction ?? "asc") &&
|
|
415
462
|
(field.scalarKind === "decimal") === (fields[i].scalarKind === "decimal") &&
|
|
416
463
|
isTextScalarKind(field.scalarKind) === isTextScalarKind(fields[i].scalarKind)
|
|
417
464
|
)
|
|
418
465
|
)
|
|
419
466
|
continue
|
|
420
|
-
|
|
467
|
+
|
|
468
|
+
const forward = index.fields.every(
|
|
469
|
+
(field, i) => (field.direction ?? "asc") === (fields[i]?.direction ?? "asc")
|
|
470
|
+
)
|
|
471
|
+
if (forward) matches.push({ index, backwards: false })
|
|
472
|
+
// Backwards, every null rank would be reversed; one field's is read apart in two ranges.
|
|
473
|
+
else if (index.fields.length === 1) matches.push({ index, backwards: true })
|
|
421
474
|
}
|
|
422
|
-
|
|
423
|
-
|
|
475
|
+
|
|
476
|
+
// The most equality filters first, then a forward walk over a backward one.
|
|
477
|
+
matches.sort(
|
|
478
|
+
(a, b) =>
|
|
479
|
+
(b.index.equality?.length ?? 0) - (a.index.equality?.length ?? 0) ||
|
|
480
|
+
Number(a.backwards) - Number(b.backwards)
|
|
481
|
+
)
|
|
482
|
+
|
|
483
|
+
const [match] = matches
|
|
484
|
+
if (!match) return undefined
|
|
485
|
+
|
|
486
|
+
const bound = sortIndexBound(match.index)
|
|
487
|
+
if (!match.backwards) return { objectTypeId, bound }
|
|
488
|
+
|
|
489
|
+
return { objectTypeId, bound, nullRank: sortIndexNullRank(match.index.fields[0]!.propertyId) }
|
|
424
490
|
}
|