kitcn 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aggregate/index.d.ts +11 -1
- package/dist/aggregate/index.js +1 -1
- package/dist/auth/generated/index.d.ts +1 -1
- package/dist/auth/generated/index.js +1 -1
- package/dist/auth/http/index.d.ts +1 -1
- package/dist/auth/index.d.ts +4 -4
- package/dist/auth/index.js +10 -9
- package/dist/auth/nextjs/index.d.ts +2 -2
- package/dist/auth/nextjs/index.js +3 -3
- package/dist/auth/start/server/index.d.ts +1 -1
- package/dist/auth/start/server/index.js +1 -1
- package/dist/{builder-CsxVc5xC.js → builder-1Vx-tEnS.js} +3 -3
- package/dist/{caller-factory-CWa0ELLD.js → caller-factory-Dd3H7j3V.js} +3 -3
- package/dist/{where-clause-compiler-DW6jy2er.d.ts → capabilities-Bem7xvGK.d.ts} +2958 -4145
- package/dist/cli.mjs +59 -40
- package/dist/{convex-plugin-D8B0oCFq.js → convex-plugin-DfOhBU9g.js} +1 -1
- package/dist/{create-schema-orm-B3f2Kc8O.js → create-schema-orm-Bu2JCiBS.js} +1 -1
- package/dist/crpc/index.d.ts +2 -2
- package/dist/crpc/index.js +3 -3
- package/dist/{customFunctions-DxEEO4Dq.js → customFunctions-BbhgdRGl.js} +2 -2
- package/dist/definitions-D9vhJ6OV.js +215 -0
- package/dist/extensions-DzjCJXYl.js +55 -0
- package/dist/filter-expression-Dydt8wS0.js +402 -0
- package/dist/{generated-contract-disabled-B-nmd7Ne.d.ts → generated-contract-disabled-BEc4d98x.d.ts} +2 -2
- package/dist/{http-types-DXOgaerG.d.ts → http-types-90mapMP9.d.ts} +2 -2
- package/dist/id-Bibg34Yb.js +37 -0
- package/dist/{local-env-yFKub75x.mjs → local-env-Du24tdbp.mjs} +2243 -2113
- package/dist/{middleware-Br5zubu7.d.ts → middleware-CE_MC6Gi.d.ts} +1 -1
- package/dist/{middleware-DUd1Sj39.js → middleware-cJDRHTRY.js} +1 -1
- package/dist/orm/aggregate-index/index.d.ts +20 -0
- package/dist/orm/aggregate-index/index.js +2136 -0
- package/dist/orm/index.d.ts +5 -4
- package/dist/orm/index.js +126 -4292
- package/dist/orm/migrations/index.d.ts +3 -0
- package/dist/orm/migrations/index.js +506 -0
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/{procedure-caller-DQxnLBS_.js → procedure-caller-DIzXfX2E.js} +2 -2
- package/dist/{procedure-name-C20pFZnk.d.ts → procedure-name-B09-cf1-.d.ts} +3 -3
- package/dist/{query-context-BzihIpnM.js → query-context-DOm5Xm3H.js} +1 -402
- package/dist/ratelimit/index.d.ts +2 -2
- package/dist/ratelimit/index.js +3 -3
- package/dist/rsc/index.d.ts +4 -4
- package/dist/rsc/index.js +4 -4
- package/dist/runtime-DfGsZtx3.js +1074 -0
- package/dist/schema-Besl-rpv.js +1411 -0
- package/dist/server/index.d.ts +4 -4
- package/dist/server/index.js +4 -4
- package/dist/{runtime-6DJxpDfb.js → table-rmJm5Qio.js} +3 -1095
- package/dist/{token-xlpENMVn.js → token-DcV_0fkF.js} +1 -1
- package/dist/{types-BHR7ZKKD.d.ts → types-In-ZZOsd.d.ts} +1 -1
- package/dist/{validators-C7LelqTN.js → validators-Dmbhsd3U.js} +1 -1
- package/dist/watcher.mjs +1 -1
- package/dist/where-clause-compiler-mohWFiG0.d.ts +2042 -0
- package/package.json +3 -2
- package/skills/kitcn/SKILL.md +19 -18
- package/skills/kitcn/references/features/aggregates.md +21 -0
- package/skills/kitcn/references/features/auth-organizations.md +3 -3
- package/skills/kitcn/references/features/auth-polar.md +2 -1
- package/skills/kitcn/references/features/auth.md +8 -1
- package/skills/kitcn/references/setup/auth.md +52 -42
- package/skills/kitcn/references/setup/index.md +1 -1
- package/skills/kitcn/references/setup/server.md +2 -1
- /package/dist/{api-entry-N3nBOlI2.js → api-entry-CkDpGYVg.js} +0 -0
- /package/dist/{context-utils-BBUtBqjN.d.ts → context-utils-DwZ3Cam1.d.ts} +0 -0
- /package/dist/{create-schema-BXrKE2YY.js → create-schema-BRFjdgRr.js} +0 -0
- /package/dist/{error-Bvo7YEhk.js → error-CMLeCadS.js} +0 -0
- /package/dist/{generated-contract-disabled-BXaz7JCE.js → generated-contract-disabled-CZa0iyV0.js} +0 -0
- /package/dist/{meta-utils-D9K4fICl.js → meta-utils-D2lqN2Lg.js} +0 -0
- /package/dist/{query-context-CNo9ffvI.d.ts → query-context-DJONf8X5.d.ts} +0 -0
- /package/dist/{query-options-CzRV4G4N.js → query-options-DUiOj01a.js} +0 -0
- /package/dist/{token-B9Bjcqug.d.ts → token-kQaqFby4.d.ts} +0 -0
- /package/dist/{transformer-yZuBWo8v.js → transformer-D8wO-kEj.js} +0 -0
- /package/dist/{types-BCl8gfGy.d.ts → types-Bf3XQex5.d.ts} +0 -0
- /package/dist/{types-DF2cg_w0.d.ts → types-MHYXboyy.d.ts} +0 -0
- /package/dist/{types-CnTpHR1F.d.ts → types-eNzQmbqM.d.ts} +0 -0
- /package/dist/{upstream-BR6sBLg3.js → upstream-BCgGZX6q.js} +0 -0
- /package/dist/{validators-BhsByJeg.d.ts → validators-V4eM2-Jg.d.ts} +0 -0
|
@@ -0,0 +1,2136 @@
|
|
|
1
|
+
import { t as DirectAggregate } from "../../runtime-DfGsZtx3.js";
|
|
2
|
+
import { a as Columns } from "../../table-rmJm5Qio.js";
|
|
3
|
+
import { Y as normalizeTemporalComparableValue, _t as usesSystemCreatedAtAlias, c as AGGREGATE_ERROR, f as createError, gt as PUBLIC_CREATED_AT_FIELD, ht as INTERNAL_CREATION_TIME_FIELD, i as AGGREGATE_STATE_TABLE, it as mapWithConcurrency, l as COUNT_ERROR, n as AGGREGATE_EXTREMA_TABLE, o as getAggregateIndexDefinitions, r as AGGREGATE_MEMBER_TABLE, s as getRankIndexDefinitions, t as AGGREGATE_BUCKET_TABLE } from "../../schema-Besl-rpv.js";
|
|
4
|
+
|
|
5
|
+
//#region src/orm/aggregate-index/runtime.ts
|
|
6
|
+
const UNDEFINED_SENTINEL = "__kitcnUndefined";
|
|
7
|
+
const FLOAT64_SIGN_BIT = 1n << 63n;
|
|
8
|
+
const FLOAT64_MASK = (1n << 64n) - 1n;
|
|
9
|
+
const DEFAULT_AGGREGATE_CARTESIAN_MAX_KEYS = 4096;
|
|
10
|
+
const DEFAULT_AGGREGATE_WORK_BUDGET = 16384;
|
|
11
|
+
const RANGE_PREFIX_WORK_UNIT_BASE = 2;
|
|
12
|
+
const AGGREGATE_BUCKET_READ_CONCURRENCY = 25;
|
|
13
|
+
const PUBLIC_ID_FIELD = "id";
|
|
14
|
+
const INTERNAL_ID_FIELD = "_id";
|
|
15
|
+
const AGGREGATE_STATE_KIND_METRIC = "metric";
|
|
16
|
+
const AGGREGATE_STATE_KIND_RANK = "rank";
|
|
17
|
+
const COUNT_STATUS_BUILDING = "BUILDING";
|
|
18
|
+
const COUNT_STATUS_READY = "READY";
|
|
19
|
+
/** Stored state is being drained before a rebuild, across several mutations. */
|
|
20
|
+
const COUNT_STATUS_CLEARING = "CLEARING";
|
|
21
|
+
const RANGE_WORK_STATE = Symbol("aggregateRangeWorkState");
|
|
22
|
+
const isPlainObject = (value) => !!value && typeof value === "object" && !Array.isArray(value);
|
|
23
|
+
const normalizeUndefined = (value) => {
|
|
24
|
+
if (Array.isArray(value)) return value.map((entry) => normalizeUndefined(entry));
|
|
25
|
+
if (isPlainObject(value)) return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, normalizeUndefined(entry)]));
|
|
26
|
+
if (value === void 0) return { [UNDEFINED_SENTINEL]: true };
|
|
27
|
+
return value;
|
|
28
|
+
};
|
|
29
|
+
const serializeStable = (value) => JSON.stringify(normalizeUndefined(value));
|
|
30
|
+
const toConstraintSet = (values) => {
|
|
31
|
+
const set = /* @__PURE__ */ new Map();
|
|
32
|
+
for (const value of values) set.set(serializeStable(value), value);
|
|
33
|
+
return set;
|
|
34
|
+
};
|
|
35
|
+
const intersectConstraintSets = (left, right) => {
|
|
36
|
+
const result = /* @__PURE__ */ new Map();
|
|
37
|
+
for (const [key, value] of left.entries()) if (right.has(key)) result.set(key, value);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
const getColumnNames = (tableConfig) => {
|
|
41
|
+
const names = new Set(Object.keys(tableConfig.table[Columns] ?? {}));
|
|
42
|
+
names.add(INTERNAL_ID_FIELD);
|
|
43
|
+
if (usesSystemCreatedAtAlias(tableConfig.table)) names.add(INTERNAL_CREATION_TIME_FIELD);
|
|
44
|
+
return names;
|
|
45
|
+
};
|
|
46
|
+
const getRelationNames = (tableConfig) => new Set(Object.keys(tableConfig.relations ?? {}));
|
|
47
|
+
const normalizeFilterFieldName = (tableConfig, fieldName) => {
|
|
48
|
+
if (fieldName === PUBLIC_ID_FIELD) return INTERNAL_ID_FIELD;
|
|
49
|
+
if (fieldName === PUBLIC_CREATED_AT_FIELD && usesSystemCreatedAtAlias(tableConfig.table)) return INTERNAL_CREATION_TIME_FIELD;
|
|
50
|
+
return fieldName;
|
|
51
|
+
};
|
|
52
|
+
const createFilterError = (codes, methodName, message) => createError(codes.FILTER_UNSUPPORTED, `${methodName} ${message}`);
|
|
53
|
+
const isRangeComparableValue = (value) => {
|
|
54
|
+
if (typeof value === "number") return Number.isFinite(value);
|
|
55
|
+
return typeof value === "string" || typeof value === "boolean";
|
|
56
|
+
};
|
|
57
|
+
const withFieldConstraint = (target, fieldName) => {
|
|
58
|
+
const existing = target.get(fieldName);
|
|
59
|
+
if (existing) return existing;
|
|
60
|
+
const next = {};
|
|
61
|
+
target.set(fieldName, next);
|
|
62
|
+
return next;
|
|
63
|
+
};
|
|
64
|
+
const normalizeAggregateComparableValue = (tableConfig, fieldName, value) => {
|
|
65
|
+
if (fieldName === INTERNAL_CREATION_TIME_FIELD) {
|
|
66
|
+
if (value instanceof Date) return value.getTime();
|
|
67
|
+
if (Array.isArray(value)) return value.map((entry) => entry instanceof Date ? entry.getTime() : entry);
|
|
68
|
+
return value;
|
|
69
|
+
}
|
|
70
|
+
return normalizeTemporalComparableValue(tableConfig.table, fieldName, value);
|
|
71
|
+
};
|
|
72
|
+
const pushConstraint = (target, fieldName, values) => {
|
|
73
|
+
const incoming = toConstraintSet(values);
|
|
74
|
+
const existing = withFieldConstraint(target, fieldName);
|
|
75
|
+
if (!existing.values) {
|
|
76
|
+
existing.values = incoming;
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
existing.values = intersectConstraintSets(existing.values, incoming);
|
|
80
|
+
};
|
|
81
|
+
const pushRangeComparison = (target, fieldName, comparison) => {
|
|
82
|
+
const existing = withFieldConstraint(target, fieldName);
|
|
83
|
+
existing.rangeComparisons ??= [];
|
|
84
|
+
existing.rangeComparisons.push(comparison);
|
|
85
|
+
};
|
|
86
|
+
const matchesRangeComparisons = (value, comparisons) => {
|
|
87
|
+
if (!isRangeComparableValue(value)) return false;
|
|
88
|
+
for (const comparison of comparisons) {
|
|
89
|
+
if (!isRangeComparableValue(comparison.value)) return false;
|
|
90
|
+
const compared = compareSortableValues(value, comparison.value);
|
|
91
|
+
if (comparison.operator === "gt" && !(compared > 0)) return false;
|
|
92
|
+
if (comparison.operator === "gte" && !(compared >= 0)) return false;
|
|
93
|
+
if (comparison.operator === "lt" && !(compared < 0)) return false;
|
|
94
|
+
if (comparison.operator === "lte" && !(compared <= 0)) return false;
|
|
95
|
+
}
|
|
96
|
+
return true;
|
|
97
|
+
};
|
|
98
|
+
const normalizeConstraints = (constraints, codes, methodName) => {
|
|
99
|
+
for (const [fieldName, constraint] of constraints.entries()) {
|
|
100
|
+
if (constraint.rangeComparisons?.length) {
|
|
101
|
+
for (const comparison of constraint.rangeComparisons) if (!isRangeComparableValue(comparison.value)) throw createFilterError(codes, methodName, `range operators on '${fieldName}' only support finite numbers, strings, and booleans.`);
|
|
102
|
+
}
|
|
103
|
+
if (constraint.values && constraint.rangeComparisons?.length) {
|
|
104
|
+
const filtered = /* @__PURE__ */ new Map();
|
|
105
|
+
for (const [stableKey, candidate] of constraint.values.entries()) if (matchesRangeComparisons(candidate, constraint.rangeComparisons)) filtered.set(stableKey, candidate);
|
|
106
|
+
constraint.values = filtered;
|
|
107
|
+
constraint.rangeComparisons = void 0;
|
|
108
|
+
}
|
|
109
|
+
if ((!constraint.values || constraint.values.size === 0) && (!constraint.rangeComparisons || constraint.rangeComparisons.length === 0)) {
|
|
110
|
+
if (constraint.values && constraint.values.size === 0) continue;
|
|
111
|
+
constraints.delete(fieldName);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
const parseFieldFilter = (tableConfig, fieldName, value, target, codes, methodName) => {
|
|
116
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
117
|
+
pushConstraint(target, fieldName, [normalizeAggregateComparableValue(tableConfig, fieldName, value)]);
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
const filter = value;
|
|
121
|
+
if (Object.hasOwn(filter, "OR") || Object.hasOwn(filter, "NOT")) throw createFilterError(codes, methodName, `does not support OR/NOT for field '${fieldName}'.`);
|
|
122
|
+
if (Object.hasOwn(filter, "AND")) {
|
|
123
|
+
const andEntries = filter.AND;
|
|
124
|
+
if (!Array.isArray(andEntries)) throw createFilterError(codes, methodName, `field-level AND for '${fieldName}' must be an array.`);
|
|
125
|
+
for (const entry of andEntries) parseFieldFilter(tableConfig, fieldName, entry, target, codes, methodName);
|
|
126
|
+
}
|
|
127
|
+
let hasRecognizedOperator = false;
|
|
128
|
+
if (Object.hasOwn(filter, "eq")) {
|
|
129
|
+
hasRecognizedOperator = true;
|
|
130
|
+
pushConstraint(target, fieldName, [normalizeAggregateComparableValue(tableConfig, fieldName, filter.eq)]);
|
|
131
|
+
}
|
|
132
|
+
if (Object.hasOwn(filter, "in")) {
|
|
133
|
+
hasRecognizedOperator = true;
|
|
134
|
+
const inValues = filter.in;
|
|
135
|
+
if (!Array.isArray(inValues)) throw createFilterError(codes, methodName, `field '${fieldName}'.in must be an array.`);
|
|
136
|
+
pushConstraint(target, fieldName, normalizeAggregateComparableValue(tableConfig, fieldName, inValues));
|
|
137
|
+
}
|
|
138
|
+
if (Object.hasOwn(filter, "isNull")) {
|
|
139
|
+
hasRecognizedOperator = true;
|
|
140
|
+
if (filter.isNull !== true) throw createFilterError(codes, methodName, `field '${fieldName}'.isNull only supports true.`);
|
|
141
|
+
pushConstraint(target, fieldName, [null]);
|
|
142
|
+
}
|
|
143
|
+
for (const operator of [
|
|
144
|
+
"gt",
|
|
145
|
+
"gte",
|
|
146
|
+
"lt",
|
|
147
|
+
"lte"
|
|
148
|
+
]) {
|
|
149
|
+
if (!Object.hasOwn(filter, operator)) continue;
|
|
150
|
+
const boundValue = filter[operator];
|
|
151
|
+
if (boundValue === void 0) continue;
|
|
152
|
+
hasRecognizedOperator = true;
|
|
153
|
+
if (boundValue === null || Array.isArray(boundValue) || isPlainObject(boundValue)) throw createFilterError(codes, methodName, `field '${fieldName}'.${operator} requires a scalar bound.`);
|
|
154
|
+
pushRangeComparison(target, fieldName, {
|
|
155
|
+
operator,
|
|
156
|
+
value: normalizeAggregateComparableValue(tableConfig, fieldName, boundValue)
|
|
157
|
+
});
|
|
158
|
+
}
|
|
159
|
+
const unsupportedOperators = Object.keys(filter).filter((operator) => ![
|
|
160
|
+
"eq",
|
|
161
|
+
"in",
|
|
162
|
+
"isNull",
|
|
163
|
+
"gt",
|
|
164
|
+
"gte",
|
|
165
|
+
"lt",
|
|
166
|
+
"lte",
|
|
167
|
+
"AND"
|
|
168
|
+
].includes(operator) && filter[operator] !== void 0);
|
|
169
|
+
if (unsupportedOperators.length > 0) throw createFilterError(codes, methodName, `does not support operators [${unsupportedOperators.join(", ")}] on '${fieldName}'.`);
|
|
170
|
+
if (!hasRecognizedOperator && !Object.hasOwn(filter, "AND")) throw createFilterError(codes, methodName, `filter for '${fieldName}' is not supported.`);
|
|
171
|
+
};
|
|
172
|
+
const constraintValuesEqual = (left, right) => {
|
|
173
|
+
if (!left && !right) return true;
|
|
174
|
+
if (!left || !right) return false;
|
|
175
|
+
if (left.size !== right.size) return false;
|
|
176
|
+
for (const key of left.keys()) if (!right.has(key)) return false;
|
|
177
|
+
return true;
|
|
178
|
+
};
|
|
179
|
+
const constraintValuesToFilter = (values) => {
|
|
180
|
+
if (values.size === 1) return values.values().next().value;
|
|
181
|
+
return { in: [...values.values()] };
|
|
182
|
+
};
|
|
183
|
+
const parseFiniteOrBranch = (options) => {
|
|
184
|
+
const { branch, tableConfig, codes, methodName } = options;
|
|
185
|
+
if (Object.hasOwn(branch, "AND") || Object.hasOwn(branch, "OR") || Object.hasOwn(branch, "NOT") || Object.hasOwn(branch, "RAW")) return null;
|
|
186
|
+
const columnNames = getColumnNames(tableConfig);
|
|
187
|
+
const relationNames = getRelationNames(tableConfig);
|
|
188
|
+
const constraints = /* @__PURE__ */ new Map();
|
|
189
|
+
for (const [key, value] of Object.entries(branch)) {
|
|
190
|
+
const normalizedKey = normalizeFilterFieldName(tableConfig, key);
|
|
191
|
+
if (!columnNames.has(normalizedKey)) {
|
|
192
|
+
if (relationNames.has(key)) throw createFilterError(codes, methodName, `does not support relation filters ('${key}') in v1.`);
|
|
193
|
+
throw createFilterError(codes, methodName, `filter field '${key}' is not recognized.`);
|
|
194
|
+
}
|
|
195
|
+
parseFieldFilter(tableConfig, normalizedKey, value, constraints, codes, methodName);
|
|
196
|
+
}
|
|
197
|
+
normalizeConstraints(constraints, codes, methodName);
|
|
198
|
+
const result = /* @__PURE__ */ new Map();
|
|
199
|
+
for (const [field, constraint] of constraints.entries()) {
|
|
200
|
+
if (constraint.rangeComparisons?.length) return null;
|
|
201
|
+
if (!constraint.values) continue;
|
|
202
|
+
result.set(field, new Map(constraint.values.entries()));
|
|
203
|
+
}
|
|
204
|
+
return result;
|
|
205
|
+
};
|
|
206
|
+
const collapseSafeOrBranches = (options) => {
|
|
207
|
+
const { branches, tableConfig, codes, methodName } = options;
|
|
208
|
+
if (branches.length === 0) return null;
|
|
209
|
+
const branchConstraints = branches.map((branch) => parseFiniteOrBranch({
|
|
210
|
+
branch,
|
|
211
|
+
tableConfig,
|
|
212
|
+
codes,
|
|
213
|
+
methodName
|
|
214
|
+
})).filter((entry) => entry !== null);
|
|
215
|
+
if (branchConstraints.length !== branches.length || branchConstraints.length === 0) return null;
|
|
216
|
+
const fields = /* @__PURE__ */ new Set();
|
|
217
|
+
for (const branch of branchConstraints) for (const field of branch.keys()) fields.add(field);
|
|
218
|
+
if (fields.size === 0) return null;
|
|
219
|
+
const varyingFields = [];
|
|
220
|
+
const first = branchConstraints[0];
|
|
221
|
+
for (const field of fields) {
|
|
222
|
+
const firstValues = first.get(field);
|
|
223
|
+
if (branchConstraints.some((branch) => !constraintValuesEqual(firstValues, branch.get(field)))) varyingFields.push(field);
|
|
224
|
+
}
|
|
225
|
+
if (varyingFields.length === 0) {
|
|
226
|
+
const collapsed = {};
|
|
227
|
+
for (const [field, values] of first.entries()) collapsed[field] = constraintValuesToFilter(values);
|
|
228
|
+
return collapsed;
|
|
229
|
+
}
|
|
230
|
+
if (varyingFields.length !== 1) return null;
|
|
231
|
+
const varyingField = varyingFields[0];
|
|
232
|
+
const unionValues = /* @__PURE__ */ new Map();
|
|
233
|
+
for (const branch of branchConstraints) {
|
|
234
|
+
const values = branch.get(varyingField);
|
|
235
|
+
if (!values || values.size === 0) return null;
|
|
236
|
+
for (const [key, value] of values.entries()) unionValues.set(key, value);
|
|
237
|
+
}
|
|
238
|
+
const collapsed = { [varyingField]: constraintValuesToFilter(unionValues) };
|
|
239
|
+
for (const [field, values] of first.entries()) {
|
|
240
|
+
if (field === varyingField) continue;
|
|
241
|
+
collapsed[field] = constraintValuesToFilter(values);
|
|
242
|
+
}
|
|
243
|
+
return collapsed;
|
|
244
|
+
};
|
|
245
|
+
const rewriteSafeOrWhere = (options) => {
|
|
246
|
+
const { where, tableConfig, codes, methodName } = options;
|
|
247
|
+
if (Object.hasOwn(where, "NOT")) throw createFilterError(codes, methodName, "NOT is not supported in no-scan aggregate/count filters.");
|
|
248
|
+
const normalized = {};
|
|
249
|
+
for (const [key, value] of Object.entries(where)) {
|
|
250
|
+
if (key === "AND") {
|
|
251
|
+
if (value === void 0) continue;
|
|
252
|
+
if (!Array.isArray(value)) throw createFilterError(codes, methodName, "AND must be an array of filter objects.");
|
|
253
|
+
normalized.AND = value.map((entry) => {
|
|
254
|
+
if (!isPlainObject(entry)) throw createFilterError(codes, methodName, "AND entries must be objects.");
|
|
255
|
+
return rewriteSafeOrWhere({
|
|
256
|
+
where: entry,
|
|
257
|
+
tableConfig,
|
|
258
|
+
codes,
|
|
259
|
+
methodName
|
|
260
|
+
});
|
|
261
|
+
});
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
if (key === "OR") continue;
|
|
265
|
+
normalized[key] = value;
|
|
266
|
+
}
|
|
267
|
+
if (!Object.hasOwn(where, "OR")) return normalized;
|
|
268
|
+
const branches = where.OR;
|
|
269
|
+
if (!Array.isArray(branches) || branches.length === 0) throw createFilterError(codes, methodName, "OR must be a non-empty array of filter objects.");
|
|
270
|
+
const collapsedOr = collapseSafeOrBranches({
|
|
271
|
+
branches: branches.map((entry) => {
|
|
272
|
+
if (!isPlainObject(entry)) throw createFilterError(codes, methodName, "OR entries must be objects.");
|
|
273
|
+
return rewriteSafeOrWhere({
|
|
274
|
+
where: entry,
|
|
275
|
+
tableConfig,
|
|
276
|
+
codes,
|
|
277
|
+
methodName
|
|
278
|
+
});
|
|
279
|
+
}),
|
|
280
|
+
tableConfig,
|
|
281
|
+
codes,
|
|
282
|
+
methodName
|
|
283
|
+
});
|
|
284
|
+
if (!collapsedOr) throw createFilterError(codes, methodName, "OR rewrite only supports finite index-plannable branches that differ on one scalar eq/in/isNull field.");
|
|
285
|
+
if (Object.keys(normalized).length === 0) return collapsedOr;
|
|
286
|
+
return { AND: [normalized, collapsedOr] };
|
|
287
|
+
};
|
|
288
|
+
const parseWhereObject = (where, tableConfig, target, codes, methodName) => {
|
|
289
|
+
if (Object.hasOwn(where, "OR") || Object.hasOwn(where, "NOT")) throw createFilterError(codes, methodName, "only supports conjunctions (object fields + AND).");
|
|
290
|
+
if (Object.hasOwn(where, "RAW")) throw createFilterError(codes, methodName, "does not support RAW filters.");
|
|
291
|
+
if (Object.hasOwn(where, "AND")) {
|
|
292
|
+
const andEntries = where.AND;
|
|
293
|
+
if (!Array.isArray(andEntries)) throw createFilterError(codes, methodName, "AND must be an array of filter objects.");
|
|
294
|
+
for (const entry of andEntries) {
|
|
295
|
+
if (!isPlainObject(entry)) throw createFilterError(codes, methodName, "AND entries must be objects.");
|
|
296
|
+
parseWhereObject(entry, tableConfig, target, codes, methodName);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
const columnNames = getColumnNames(tableConfig);
|
|
300
|
+
const relationNames = getRelationNames(tableConfig);
|
|
301
|
+
for (const [key, value] of Object.entries(where)) {
|
|
302
|
+
if (key === "AND") continue;
|
|
303
|
+
const normalizedKey = normalizeFilterFieldName(tableConfig, key);
|
|
304
|
+
if (columnNames.has(normalizedKey)) {
|
|
305
|
+
parseFieldFilter(tableConfig, normalizedKey, value, target, codes, methodName);
|
|
306
|
+
continue;
|
|
307
|
+
}
|
|
308
|
+
if (relationNames.has(key)) throw createFilterError(codes, methodName, `does not support relation filters ('${key}') in v1.`);
|
|
309
|
+
throw createFilterError(codes, methodName, `filter field '${key}' is not recognized.`);
|
|
310
|
+
}
|
|
311
|
+
};
|
|
312
|
+
const constraintsToValueRecord = (constraints) => {
|
|
313
|
+
const record = {};
|
|
314
|
+
for (const [field, constraint] of constraints.entries()) {
|
|
315
|
+
if (!constraint.values) continue;
|
|
316
|
+
record[field] = [...constraint.values.values()];
|
|
317
|
+
}
|
|
318
|
+
return record;
|
|
319
|
+
};
|
|
320
|
+
const supportsMetric = (index, metric) => {
|
|
321
|
+
if (metric.kind === "count") return true;
|
|
322
|
+
if (metric.kind === "countField") return index.countFields.includes(metric.field);
|
|
323
|
+
if (metric.kind === "sum") return index.sumFields.includes(metric.field) || index.avgFields.includes(metric.field);
|
|
324
|
+
if (metric.kind === "avg") return index.avgFields.includes(metric.field);
|
|
325
|
+
if (metric.kind === "min") return index.minFields.includes(metric.field);
|
|
326
|
+
return index.maxFields.includes(metric.field);
|
|
327
|
+
};
|
|
328
|
+
const pickAggregateIndex = (indexDefinitions, constrainedFields, metric) => {
|
|
329
|
+
let best = null;
|
|
330
|
+
for (const definition of indexDefinitions) {
|
|
331
|
+
if (!definition.fields.every((field) => constrainedFields.has(field))) continue;
|
|
332
|
+
const indexFieldSet = new Set(definition.fields);
|
|
333
|
+
if ([...constrainedFields].some((field) => !indexFieldSet.has(field))) continue;
|
|
334
|
+
if (!supportsMetric(definition, metric)) continue;
|
|
335
|
+
if (!best || definition.fields.length > best.fields.length) best = definition;
|
|
336
|
+
}
|
|
337
|
+
return best;
|
|
338
|
+
};
|
|
339
|
+
const pickRangeAggregateIndex = (indexDefinitions, constraints, constrainedFields, rangeFieldName, metric) => {
|
|
340
|
+
let best = null;
|
|
341
|
+
for (const definition of indexDefinitions) {
|
|
342
|
+
if (!supportsMetric(definition, metric)) continue;
|
|
343
|
+
const usesImplicitCreationTimeSuffix = rangeFieldName === INTERNAL_CREATION_TIME_FIELD;
|
|
344
|
+
const rangeFieldPosition = usesImplicitCreationTimeSuffix ? definition.fields.length : definition.fields.indexOf(rangeFieldName);
|
|
345
|
+
if (rangeFieldPosition < 0) continue;
|
|
346
|
+
const indexFieldSet = new Set(definition.fields);
|
|
347
|
+
if (usesImplicitCreationTimeSuffix) indexFieldSet.add(INTERNAL_CREATION_TIME_FIELD);
|
|
348
|
+
if ([...constrainedFields].some((field) => !indexFieldSet.has(field))) continue;
|
|
349
|
+
const prefixFields = definition.fields.slice(0, rangeFieldPosition);
|
|
350
|
+
if (prefixFields.some((field) => {
|
|
351
|
+
return !constraints.get(field)?.values;
|
|
352
|
+
})) continue;
|
|
353
|
+
if (prefixFields.some((field) => {
|
|
354
|
+
const fieldConstraint = constraints.get(field);
|
|
355
|
+
return Boolean(fieldConstraint?.rangeComparisons?.length);
|
|
356
|
+
})) continue;
|
|
357
|
+
if (!best || prefixFields.length > best.prefixFields.length || prefixFields.length === best.prefixFields.length && definition.fields.length > best.definition.fields.length) best = {
|
|
358
|
+
definition,
|
|
359
|
+
prefixFields
|
|
360
|
+
};
|
|
361
|
+
}
|
|
362
|
+
return best;
|
|
363
|
+
};
|
|
364
|
+
const buildCandidateKeys = (fields, constraints, index = 0, current = [], output = []) => {
|
|
365
|
+
if (index >= fields.length) {
|
|
366
|
+
output.push([...current]);
|
|
367
|
+
return output;
|
|
368
|
+
}
|
|
369
|
+
const values = constraints[fields[index]] ?? [];
|
|
370
|
+
for (const value of values) {
|
|
371
|
+
current.push(value);
|
|
372
|
+
buildCandidateKeys(fields, constraints, index + 1, current, output);
|
|
373
|
+
current.pop();
|
|
374
|
+
}
|
|
375
|
+
return output;
|
|
376
|
+
};
|
|
377
|
+
const getAggregateCartesianMaxKeys = (tableConfig) => {
|
|
378
|
+
const value = tableConfig.defaults?.aggregateCartesianMaxKeys;
|
|
379
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 1) return Math.floor(value);
|
|
380
|
+
return DEFAULT_AGGREGATE_CARTESIAN_MAX_KEYS;
|
|
381
|
+
};
|
|
382
|
+
const getAggregateWorkBudget = (tableConfig) => {
|
|
383
|
+
const value = tableConfig.defaults?.aggregateWorkBudget;
|
|
384
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 1) return Math.floor(value);
|
|
385
|
+
return DEFAULT_AGGREGATE_WORK_BUDGET;
|
|
386
|
+
};
|
|
387
|
+
const countCartesianCombinations = (fields, constraints, maxCount) => {
|
|
388
|
+
if (fields.length === 0) return 1;
|
|
389
|
+
let total = 1;
|
|
390
|
+
for (const field of fields) {
|
|
391
|
+
const values = constraints[field] ?? [];
|
|
392
|
+
if (values.length === 0) return 0;
|
|
393
|
+
const threshold = Math.max(1, maxCount);
|
|
394
|
+
const maxBeforeMultiply = Math.floor(threshold / values.length);
|
|
395
|
+
if (total > maxBeforeMultiply) return threshold + 1;
|
|
396
|
+
total *= values.length;
|
|
397
|
+
}
|
|
398
|
+
return total;
|
|
399
|
+
};
|
|
400
|
+
const enforceCartesianExpansionGuards = (options) => {
|
|
401
|
+
const cartesianMaxKeys = getAggregateCartesianMaxKeys(options.tableConfig);
|
|
402
|
+
const workBudget = getAggregateWorkBudget(options.tableConfig);
|
|
403
|
+
const workUnitsPerCombination = Math.max(1, options.workUnitsPerCombination);
|
|
404
|
+
const maxCombinationsByWork = Math.max(1, Math.floor(workBudget / workUnitsPerCombination));
|
|
405
|
+
const maxTrackedCombinations = Math.max(cartesianMaxKeys, maxCombinationsByWork);
|
|
406
|
+
const combinations = countCartesianCombinations(options.fields, options.fieldValues, maxTrackedCombinations);
|
|
407
|
+
if (combinations === 0) return;
|
|
408
|
+
if (combinations > cartesianMaxKeys) throw createFilterError(options.codes, options.methodName, `expands IN filters to ${combinations} key combinations on aggregateIndex '${options.indexName}', exceeding aggregateCartesianMaxKeys (${cartesianMaxKeys}). Reduce IN list sizes, split the query, add a narrower aggregateIndex, or increase defineSchema(..., { defaults: { aggregateCartesianMaxKeys } }).`);
|
|
409
|
+
const estimatedWork = combinations * workUnitsPerCombination;
|
|
410
|
+
if (estimatedWork > workBudget) throw createFilterError(options.codes, options.methodName, `estimated ${options.workLabel} is ${estimatedWork} units on aggregateIndex '${options.indexName}', exceeding aggregateWorkBudget (${workBudget}). Reduce IN fan-out, split the query, or increase defineSchema(..., { defaults: { aggregateWorkBudget } }).`);
|
|
411
|
+
};
|
|
412
|
+
const hasLogicalOr = (where) => {
|
|
413
|
+
if (Object.hasOwn(where, "OR")) return true;
|
|
414
|
+
const andEntries = where.AND;
|
|
415
|
+
if (!Array.isArray(andEntries)) return false;
|
|
416
|
+
return andEntries.some((entry) => isPlainObject(entry) && hasLogicalOr(entry));
|
|
417
|
+
};
|
|
418
|
+
const combineDnfBranches = (options) => {
|
|
419
|
+
const { left, right, maxBranches, codes, methodName } = options;
|
|
420
|
+
if (left.length === 0 || right.length === 0) return [];
|
|
421
|
+
const threshold = Math.max(1, maxBranches);
|
|
422
|
+
const maxBeforeMultiply = Math.floor(threshold / right.length);
|
|
423
|
+
if (left.length > maxBeforeMultiply) throw createFilterError(codes, methodName, `OR rewrite expands to ${left.length * right.length} branches, exceeding aggregateCartesianMaxKeys (${threshold}). Reduce OR fan-out, split the query, or increase defineSchema(..., { defaults: { aggregateCartesianMaxKeys } }).`);
|
|
424
|
+
const combined = [];
|
|
425
|
+
for (const leftBranch of left) for (const rightBranch of right) combined.push([...leftBranch, ...rightBranch]);
|
|
426
|
+
return combined;
|
|
427
|
+
};
|
|
428
|
+
const expandWhereToFiniteDnf = (options) => {
|
|
429
|
+
const { where, maxBranches, codes, methodName } = options;
|
|
430
|
+
if (Object.hasOwn(where, "NOT")) throw createFilterError(codes, methodName, "NOT is not supported in no-scan aggregate/count filters.");
|
|
431
|
+
if (Object.hasOwn(where, "RAW")) throw createFilterError(codes, methodName, "does not support RAW filters.");
|
|
432
|
+
const scalarClause = {};
|
|
433
|
+
for (const [key, value] of Object.entries(where)) {
|
|
434
|
+
if (key === "AND" || key === "OR") continue;
|
|
435
|
+
scalarClause[key] = value;
|
|
436
|
+
}
|
|
437
|
+
let branches = [[]];
|
|
438
|
+
if (Object.keys(scalarClause).length > 0) branches = branches.map((branch) => [...branch, scalarClause]);
|
|
439
|
+
if (Object.hasOwn(where, "AND")) {
|
|
440
|
+
const andEntries = where.AND;
|
|
441
|
+
if (!Array.isArray(andEntries)) throw createFilterError(codes, methodName, "AND must be an array of filter objects.");
|
|
442
|
+
for (const entry of andEntries) {
|
|
443
|
+
if (!isPlainObject(entry)) throw createFilterError(codes, methodName, "AND entries must be objects.");
|
|
444
|
+
const entryBranches = expandWhereToFiniteDnf({
|
|
445
|
+
where: entry,
|
|
446
|
+
maxBranches,
|
|
447
|
+
codes,
|
|
448
|
+
methodName
|
|
449
|
+
});
|
|
450
|
+
branches = combineDnfBranches({
|
|
451
|
+
left: branches,
|
|
452
|
+
right: entryBranches,
|
|
453
|
+
maxBranches,
|
|
454
|
+
codes,
|
|
455
|
+
methodName
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
if (Object.hasOwn(where, "OR")) {
|
|
460
|
+
const orEntries = where.OR;
|
|
461
|
+
if (!Array.isArray(orEntries) || orEntries.length === 0) throw createFilterError(codes, methodName, "OR must be a non-empty array of filter objects.");
|
|
462
|
+
const orBranches = [];
|
|
463
|
+
for (const entry of orEntries) {
|
|
464
|
+
if (!isPlainObject(entry)) throw createFilterError(codes, methodName, "OR entries must be objects.");
|
|
465
|
+
const expanded = expandWhereToFiniteDnf({
|
|
466
|
+
where: entry,
|
|
467
|
+
maxBranches,
|
|
468
|
+
codes,
|
|
469
|
+
methodName
|
|
470
|
+
});
|
|
471
|
+
orBranches.push(...expanded);
|
|
472
|
+
if (orBranches.length > maxBranches) throw createFilterError(codes, methodName, `OR rewrite expands to ${orBranches.length} branches, exceeding aggregateCartesianMaxKeys (${maxBranches}). Reduce OR fan-out, split the query, or increase defineSchema(..., { defaults: { aggregateCartesianMaxKeys } }).`);
|
|
473
|
+
}
|
|
474
|
+
branches = combineDnfBranches({
|
|
475
|
+
left: branches,
|
|
476
|
+
right: orBranches,
|
|
477
|
+
maxBranches,
|
|
478
|
+
codes,
|
|
479
|
+
methodName
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
return branches;
|
|
483
|
+
};
|
|
484
|
+
const branchToWhere = (branch) => {
|
|
485
|
+
if (branch.length === 0) return {};
|
|
486
|
+
if (branch.length === 1) return branch[0];
|
|
487
|
+
return { AND: branch };
|
|
488
|
+
};
|
|
489
|
+
const compileFiniteOrUnionPlan = (options) => {
|
|
490
|
+
const { tableConfig, where, metric, aggregateIndexes, codes, methodName, throwNotIndexed } = options;
|
|
491
|
+
if (!hasLogicalOr(where)) return null;
|
|
492
|
+
const cartesianMaxKeys = getAggregateCartesianMaxKeys(tableConfig);
|
|
493
|
+
const workBudget = getAggregateWorkBudget(tableConfig);
|
|
494
|
+
const branches = expandWhereToFiniteDnf({
|
|
495
|
+
where,
|
|
496
|
+
maxBranches: Math.max(1, Math.min(cartesianMaxKeys, workBudget)),
|
|
497
|
+
codes,
|
|
498
|
+
methodName
|
|
499
|
+
});
|
|
500
|
+
if (branches.length === 0) throw createFilterError(codes, methodName, "OR must include at least one branch.");
|
|
501
|
+
const dedupedKeyCandidates = /* @__PURE__ */ new Map();
|
|
502
|
+
let selectedIndexName = null;
|
|
503
|
+
let selectedIndexFields = null;
|
|
504
|
+
let totalBranchCandidates = 0;
|
|
505
|
+
for (const branch of branches) {
|
|
506
|
+
const branchWhere = branchToWhere(branch);
|
|
507
|
+
const constraints = /* @__PURE__ */ new Map();
|
|
508
|
+
parseWhereObject(branchWhere, tableConfig, constraints, codes, methodName);
|
|
509
|
+
normalizeConstraints(constraints, codes, methodName);
|
|
510
|
+
if ([...constraints.entries()].filter(([, constraint]) => (constraint.rangeComparisons?.length ?? 0) > 0).map(([field]) => field).length > 0) throw createFilterError(codes, methodName, "OR rewrite supports finite eq/in/isNull branches only in v1. Range operators inside OR are not supported.");
|
|
511
|
+
const matchedIndex = pickAggregateIndex(aggregateIndexes, new Set(constraints.keys()), metric);
|
|
512
|
+
if (!matchedIndex) throwNotIndexed();
|
|
513
|
+
const exactIndex = matchedIndex;
|
|
514
|
+
if (selectedIndexName === null || selectedIndexFields === null) {
|
|
515
|
+
selectedIndexName = exactIndex.name;
|
|
516
|
+
selectedIndexFields = exactIndex.fields;
|
|
517
|
+
} else if (selectedIndexName !== exactIndex.name || selectedIndexFields.length !== exactIndex.fields.length || selectedIndexFields.some((field, index) => field !== exactIndex.fields[index])) throw createFilterError(codes, methodName, `OR branches must resolve to a single aggregateIndex. Found incompatible branches between '${selectedIndexName}' and '${exactIndex.name}'.`);
|
|
518
|
+
const rawFieldValues = constraintsToValueRecord(constraints);
|
|
519
|
+
const fieldValues = Object.fromEntries(exactIndex.fields.map((field) => [field, rawFieldValues[field] ?? []]));
|
|
520
|
+
enforceCartesianExpansionGuards({
|
|
521
|
+
tableConfig,
|
|
522
|
+
codes,
|
|
523
|
+
methodName,
|
|
524
|
+
indexName: exactIndex.name,
|
|
525
|
+
fields: exactIndex.fields,
|
|
526
|
+
fieldValues,
|
|
527
|
+
workUnitsPerCombination: 1,
|
|
528
|
+
workLabel: "OR-branch key lookup work"
|
|
529
|
+
});
|
|
530
|
+
if (Object.values(fieldValues).some((values) => values.length === 0)) continue;
|
|
531
|
+
const branchCandidates = buildCandidateKeys(exactIndex.fields, fieldValues);
|
|
532
|
+
totalBranchCandidates += branchCandidates.length;
|
|
533
|
+
for (const keyParts of branchCandidates) dedupedKeyCandidates.set(serializeCountKeyParts(keyParts), keyParts);
|
|
534
|
+
}
|
|
535
|
+
if (!selectedIndexName || !selectedIndexFields) throw createFilterError(codes, methodName, "OR rewrite did not resolve to any index-plannable branch.");
|
|
536
|
+
const uniqueKeyCount = dedupedKeyCandidates.size;
|
|
537
|
+
if (uniqueKeyCount > cartesianMaxKeys) throw createFilterError(codes, methodName, `OR rewrite expands to ${uniqueKeyCount} unique key combinations on aggregateIndex '${selectedIndexName}', exceeding aggregateCartesianMaxKeys (${cartesianMaxKeys}). Reduce OR fan-out, split the query, or increase defineSchema(..., { defaults: { aggregateCartesianMaxKeys } }).`);
|
|
538
|
+
const estimatedWork = branches.length + totalBranchCandidates;
|
|
539
|
+
if (estimatedWork > workBudget) throw createFilterError(codes, methodName, `estimated OR rewrite work is ${estimatedWork} units on aggregateIndex '${selectedIndexName}', exceeding aggregateWorkBudget (${workBudget}). Reduce OR fan-out, split the query, or increase defineSchema(..., { defaults: { aggregateWorkBudget } }).`);
|
|
540
|
+
return {
|
|
541
|
+
tableName: tableConfig.name,
|
|
542
|
+
indexName: selectedIndexName,
|
|
543
|
+
indexFields: selectedIndexFields,
|
|
544
|
+
fieldValues: {},
|
|
545
|
+
keyCandidates: [...dedupedKeyCandidates.values()],
|
|
546
|
+
rangeConstraint: null,
|
|
547
|
+
postFieldValues: {},
|
|
548
|
+
metric
|
|
549
|
+
};
|
|
550
|
+
};
|
|
551
|
+
const serializeCountKeyParts = (parts) => serializeStable(parts);
|
|
552
|
+
const compileAggregatePlan = (tableConfig, where, metric, codes, methodName) => {
|
|
553
|
+
const normalizedWhere = where ?? {};
|
|
554
|
+
if (!isPlainObject(normalizedWhere)) throw createFilterError(codes, methodName, "where must be an object filter.");
|
|
555
|
+
const aggregateIndexes = getAggregateIndexDefinitions(tableConfig);
|
|
556
|
+
const throwNotIndexed = (fieldList) => {
|
|
557
|
+
if (metric.kind === "count") throw createError(codes.NOT_INDEXED, `No matching aggregateIndex found for count() on '${tableConfig.name}'. Add an aggregateIndex for [${fieldList}] and run aggregateBackfill.`);
|
|
558
|
+
if (metric.kind === "countField") throw createError(codes.NOT_INDEXED, `No matching aggregateIndex found for count(select: { ${metric.field}: true }) on '${tableConfig.name}'. Declare aggregateIndex(...).on(...).count(...) and run aggregateBackfill.`);
|
|
559
|
+
if (metric.kind === "avg") throw createError(codes.NOT_INDEXED, `No matching aggregateIndex found for avg('${metric.field}') on '${tableConfig.name}'. Declare aggregateIndex(...).on(...).avg(...) (or .all() for unfiltered) and run aggregateBackfill.`);
|
|
560
|
+
throw createError(codes.NOT_INDEXED, `No matching aggregateIndex found for ${metric.kind}('${metric.field}') on '${tableConfig.name}'. Declare aggregateIndex(...).on(...).${metric.kind}(...) (or .all() for unfiltered) and run aggregateBackfill.`);
|
|
561
|
+
};
|
|
562
|
+
const finiteOrUnionPlan = compileFiniteOrUnionPlan({
|
|
563
|
+
tableConfig,
|
|
564
|
+
where: normalizedWhere,
|
|
565
|
+
metric,
|
|
566
|
+
aggregateIndexes,
|
|
567
|
+
codes,
|
|
568
|
+
methodName,
|
|
569
|
+
throwNotIndexed: () => throwNotIndexed("OR branches")
|
|
570
|
+
});
|
|
571
|
+
if (finiteOrUnionPlan) return finiteOrUnionPlan;
|
|
572
|
+
const constraints = /* @__PURE__ */ new Map();
|
|
573
|
+
parseWhereObject(rewriteSafeOrWhere({
|
|
574
|
+
where: normalizedWhere,
|
|
575
|
+
tableConfig,
|
|
576
|
+
codes,
|
|
577
|
+
methodName
|
|
578
|
+
}), tableConfig, constraints, codes, methodName);
|
|
579
|
+
normalizeConstraints(constraints, codes, methodName);
|
|
580
|
+
const constrainedFields = new Set(constraints.keys());
|
|
581
|
+
const rangeFields = [...constraints.entries()].filter(([, constraint]) => (constraint.rangeComparisons?.length ?? 0) > 0).map(([field]) => field);
|
|
582
|
+
if (rangeFields.length > 1) throw createFilterError(codes, methodName, `supports range operators on at most one field in v1. Found [${rangeFields.join(", ")}].`);
|
|
583
|
+
const fieldList = [...constrainedFields].join(", ");
|
|
584
|
+
if (rangeFields.length === 0) {
|
|
585
|
+
const matchedIndex = pickAggregateIndex(aggregateIndexes, constrainedFields, metric);
|
|
586
|
+
if (!matchedIndex) throwNotIndexed(fieldList);
|
|
587
|
+
const exactIndex = matchedIndex;
|
|
588
|
+
const rawFieldValues = constraintsToValueRecord(constraints);
|
|
589
|
+
const fieldValues = Object.fromEntries(exactIndex.fields.map((field) => [field, rawFieldValues[field] ?? []]));
|
|
590
|
+
enforceCartesianExpansionGuards({
|
|
591
|
+
tableConfig,
|
|
592
|
+
codes,
|
|
593
|
+
methodName,
|
|
594
|
+
indexName: exactIndex.name,
|
|
595
|
+
fields: exactIndex.fields,
|
|
596
|
+
fieldValues,
|
|
597
|
+
workUnitsPerCombination: 1,
|
|
598
|
+
workLabel: "key lookup work"
|
|
599
|
+
});
|
|
600
|
+
for (const values of Object.values(fieldValues)) if (values.length === 0) return {
|
|
601
|
+
tableName: tableConfig.name,
|
|
602
|
+
indexName: exactIndex.name,
|
|
603
|
+
indexFields: exactIndex.fields,
|
|
604
|
+
fieldValues,
|
|
605
|
+
rangeConstraint: null,
|
|
606
|
+
postFieldValues: {},
|
|
607
|
+
metric
|
|
608
|
+
};
|
|
609
|
+
return {
|
|
610
|
+
tableName: tableConfig.name,
|
|
611
|
+
indexName: exactIndex.name,
|
|
612
|
+
indexFields: exactIndex.fields,
|
|
613
|
+
fieldValues,
|
|
614
|
+
rangeConstraint: null,
|
|
615
|
+
postFieldValues: {},
|
|
616
|
+
metric
|
|
617
|
+
};
|
|
618
|
+
}
|
|
619
|
+
const rangeFieldName = rangeFields[0];
|
|
620
|
+
const rangeComparisons = constraints.get(rangeFieldName)?.rangeComparisons ?? [];
|
|
621
|
+
const matchedRangeIndex = pickRangeAggregateIndex(aggregateIndexes, constraints, constrainedFields, rangeFieldName, metric);
|
|
622
|
+
if (!matchedRangeIndex) throwNotIndexed(fieldList);
|
|
623
|
+
const rangeIndex = matchedRangeIndex;
|
|
624
|
+
const prefixFieldValues = Object.fromEntries(rangeIndex.prefixFields.map((field) => [field, [...constraints.get(field)?.values?.values() ?? []]]));
|
|
625
|
+
const postFieldValues = {};
|
|
626
|
+
for (const field of rangeIndex.definition.fields) {
|
|
627
|
+
if (field === rangeFieldName || rangeIndex.prefixFields.includes(field)) continue;
|
|
628
|
+
const values = constraints.get(field)?.values;
|
|
629
|
+
if (!values) continue;
|
|
630
|
+
postFieldValues[field] = [...values.values()];
|
|
631
|
+
}
|
|
632
|
+
const rangeWorkUnits = RANGE_PREFIX_WORK_UNIT_BASE + Object.keys(postFieldValues).length;
|
|
633
|
+
enforceCartesianExpansionGuards({
|
|
634
|
+
tableConfig,
|
|
635
|
+
codes,
|
|
636
|
+
methodName,
|
|
637
|
+
indexName: rangeIndex.definition.name,
|
|
638
|
+
fields: rangeIndex.prefixFields,
|
|
639
|
+
fieldValues: prefixFieldValues,
|
|
640
|
+
workUnitsPerCombination: rangeWorkUnits,
|
|
641
|
+
workLabel: "range-prefix work"
|
|
642
|
+
});
|
|
643
|
+
return {
|
|
644
|
+
tableName: tableConfig.name,
|
|
645
|
+
indexName: rangeIndex.definition.name,
|
|
646
|
+
indexFields: rangeIndex.definition.fields,
|
|
647
|
+
fieldValues: prefixFieldValues,
|
|
648
|
+
rangeConstraint: {
|
|
649
|
+
fieldName: rangeFieldName,
|
|
650
|
+
comparisons: rangeComparisons,
|
|
651
|
+
prefixFields: rangeIndex.prefixFields,
|
|
652
|
+
workBudget: getAggregateWorkBudget(tableConfig),
|
|
653
|
+
filterErrorCode: codes.FILTER_UNSUPPORTED,
|
|
654
|
+
methodName
|
|
655
|
+
},
|
|
656
|
+
postFieldValues,
|
|
657
|
+
metric
|
|
658
|
+
};
|
|
659
|
+
};
|
|
660
|
+
const compileCountQueryPlan = (tableConfig, where) => {
|
|
661
|
+
const plan = compileAggregatePlan(tableConfig, where, { kind: "count" }, COUNT_ERROR, "count()");
|
|
662
|
+
return {
|
|
663
|
+
tableName: plan.tableName,
|
|
664
|
+
indexName: plan.indexName,
|
|
665
|
+
indexFields: plan.indexFields,
|
|
666
|
+
fieldValues: plan.fieldValues,
|
|
667
|
+
keyCandidates: plan.keyCandidates,
|
|
668
|
+
rangeConstraint: plan.rangeConstraint,
|
|
669
|
+
postFieldValues: plan.postFieldValues
|
|
670
|
+
};
|
|
671
|
+
};
|
|
672
|
+
const compileCountFieldQueryPlan = (tableConfig, where, field) => compileAggregatePlan(tableConfig, where, {
|
|
673
|
+
kind: "countField",
|
|
674
|
+
field
|
|
675
|
+
}, COUNT_ERROR, `count(select: { ${field}: true })`);
|
|
676
|
+
const compileAggregateQueryPlan = (tableConfig, where, metric) => compileAggregatePlan(tableConfig, where, metric, AGGREGATE_ERROR, (() => {
|
|
677
|
+
if (metric.kind === "countField") return `count(select: { ${metric.field}: true })`;
|
|
678
|
+
if (metric.kind === "avg") return `avg('${metric.field}')`;
|
|
679
|
+
return `${metric.kind}()`;
|
|
680
|
+
})());
|
|
681
|
+
const deepEquals = (left, right) => serializeStable(left) === serializeStable(right);
|
|
682
|
+
const normalizeSumValues = (values) => {
|
|
683
|
+
const output = {};
|
|
684
|
+
for (const [field, value] of Object.entries(values)) if (typeof value === "number" && Number.isFinite(value)) output[field] = value;
|
|
685
|
+
return output;
|
|
686
|
+
};
|
|
687
|
+
const normalizeNonNullCountValues = (values) => {
|
|
688
|
+
const output = {};
|
|
689
|
+
for (const [field, value] of Object.entries(values)) if (typeof value === "number" && Number.isFinite(value)) output[field] = value;
|
|
690
|
+
return output;
|
|
691
|
+
};
|
|
692
|
+
const normalizeExtremaValues = (values) => ({ ...values });
|
|
693
|
+
const mergeSumValues = (baseValues, deltaValues) => {
|
|
694
|
+
const merged = { ...baseValues };
|
|
695
|
+
for (const [field, delta] of Object.entries(deltaValues)) {
|
|
696
|
+
const next = (merged[field] ?? 0) + delta;
|
|
697
|
+
if (next === 0) {
|
|
698
|
+
delete merged[field];
|
|
699
|
+
continue;
|
|
700
|
+
}
|
|
701
|
+
merged[field] = next;
|
|
702
|
+
}
|
|
703
|
+
return merged;
|
|
704
|
+
};
|
|
705
|
+
const mergeCountValues = (baseValues, deltaValues) => {
|
|
706
|
+
const merged = { ...baseValues };
|
|
707
|
+
for (const [field, delta] of Object.entries(deltaValues)) {
|
|
708
|
+
const next = (merged[field] ?? 0) + delta;
|
|
709
|
+
if (next === 0) {
|
|
710
|
+
delete merged[field];
|
|
711
|
+
continue;
|
|
712
|
+
}
|
|
713
|
+
merged[field] = next;
|
|
714
|
+
}
|
|
715
|
+
return merged;
|
|
716
|
+
};
|
|
717
|
+
const negateSumValues = (values) => Object.fromEntries(Object.entries(values).map(([field, value]) => [field, -value]));
|
|
718
|
+
const negateCountValues = (values) => Object.fromEntries(Object.entries(values).map(([field, value]) => [field, -value]));
|
|
719
|
+
const listBucketsByHash = async (db, tableName, indexName, keyHash) => await db.query(AGGREGATE_BUCKET_TABLE).withIndex("by_table_index_hash", (q) => q.eq("tableKey", tableName).eq("indexName", indexName).eq("keyHash", keyHash)).collect();
|
|
720
|
+
const getBucketByKey = async (db, tableName, indexName, keyParts) => {
|
|
721
|
+
return (await listBucketsByHash(db, tableName, indexName, serializeCountKeyParts(keyParts))).find((bucket) => deepEquals(bucket.keyParts, keyParts)) ?? null;
|
|
722
|
+
};
|
|
723
|
+
const listBucketsByHashPrefix = async (db, tableName, indexName, scan, limit) => await db.query(AGGREGATE_BUCKET_TABLE).withIndex("by_table_index_hash", (q) => {
|
|
724
|
+
const scoped = q.eq("tableKey", tableName).eq("indexName", indexName);
|
|
725
|
+
return (scan.after === void 0 ? scoped.gte("keyHash", scan.start) : scoped.gt("keyHash", scan.after)).lt("keyHash", scan.end);
|
|
726
|
+
}).take(limit);
|
|
727
|
+
const getExtremaByValue = async (db, tableName, indexName, keyHash, fieldName, value) => {
|
|
728
|
+
const valueHash = serializeStable(value);
|
|
729
|
+
return (await db.query(AGGREGATE_EXTREMA_TABLE).withIndex("by_table_index_hash_field_value", (q) => q.eq("tableKey", tableName).eq("indexName", indexName).eq("keyHash", keyHash).eq("fieldName", fieldName).eq("valueHash", valueHash)).collect()).find((row) => deepEquals(row.value, value)) ?? null;
|
|
730
|
+
};
|
|
731
|
+
const takeMembersForIndex = async (db, tableName, indexName, limit) => await db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index", (q) => q.eq("kind", AGGREGATE_STATE_KIND_METRIC).eq("tableKey", tableName).eq("indexName", indexName)).take(limit);
|
|
732
|
+
const takeBucketsForIndex = async (db, tableName, indexName, limit) => await db.query(AGGREGATE_BUCKET_TABLE).withIndex("by_table_index", (q) => q.eq("tableKey", tableName).eq("indexName", indexName)).take(limit);
|
|
733
|
+
const takeExtremaForIndex = async (db, tableName, indexName, limit) => await db.query(AGGREGATE_EXTREMA_TABLE).withIndex("by_table_index", (q) => q.eq("tableKey", tableName).eq("indexName", indexName)).take(limit);
|
|
734
|
+
const applyBucketDelta = async (db, tableName, indexName, keyParts, deltaCount, deltaSums, deltaNonNullCounts) => {
|
|
735
|
+
if (deltaCount === 0 && Object.keys(deltaSums).length === 0 && Object.keys(deltaNonNullCounts).length === 0) return;
|
|
736
|
+
const existing = await getBucketByKey(db, tableName, indexName, keyParts);
|
|
737
|
+
const now = Date.now();
|
|
738
|
+
if (!existing) {
|
|
739
|
+
if (deltaCount < 0) return;
|
|
740
|
+
await db.insert(AGGREGATE_BUCKET_TABLE, {
|
|
741
|
+
tableKey: tableName,
|
|
742
|
+
indexName,
|
|
743
|
+
keyHash: serializeCountKeyParts(keyParts),
|
|
744
|
+
keyParts,
|
|
745
|
+
count: deltaCount,
|
|
746
|
+
sumValues: deltaSums,
|
|
747
|
+
nonNullCountValues: deltaNonNullCounts,
|
|
748
|
+
updatedAt: now
|
|
749
|
+
});
|
|
750
|
+
return;
|
|
751
|
+
}
|
|
752
|
+
const nextCount = existing.count + deltaCount;
|
|
753
|
+
if (nextCount <= 0) {
|
|
754
|
+
await db.delete(AGGREGATE_BUCKET_TABLE, existing._id);
|
|
755
|
+
return;
|
|
756
|
+
}
|
|
757
|
+
const nextSumValues = mergeSumValues(normalizeSumValues(existing.sumValues), deltaSums);
|
|
758
|
+
const nextNonNullCountValues = mergeCountValues(normalizeNonNullCountValues(existing.nonNullCountValues), deltaNonNullCounts);
|
|
759
|
+
await db.patch(AGGREGATE_BUCKET_TABLE, existing._id, {
|
|
760
|
+
count: nextCount,
|
|
761
|
+
sumValues: nextSumValues,
|
|
762
|
+
nonNullCountValues: nextNonNullCountValues,
|
|
763
|
+
updatedAt: now
|
|
764
|
+
});
|
|
765
|
+
};
|
|
766
|
+
const encodeNumberSortKey = (value) => {
|
|
767
|
+
const view = /* @__PURE__ */ new DataView(/* @__PURE__ */ new ArrayBuffer(8));
|
|
768
|
+
view.setFloat64(0, value);
|
|
769
|
+
let bits = view.getBigUint64(0);
|
|
770
|
+
if ((bits & FLOAT64_SIGN_BIT) !== 0n) bits = ~bits & FLOAT64_MASK;
|
|
771
|
+
else bits |= FLOAT64_SIGN_BIT;
|
|
772
|
+
return bits.toString(16).padStart(16, "0");
|
|
773
|
+
};
|
|
774
|
+
const toComparableSortKey = (value) => {
|
|
775
|
+
if (typeof value === "number") {
|
|
776
|
+
if (!Number.isFinite(value)) throw createError(AGGREGATE_ERROR.FILTER_UNSUPPORTED, `Aggregate min/max only support finite numbers, strings, and booleans. Received ${String(value)}.`);
|
|
777
|
+
return `n:${encodeNumberSortKey(value)}`;
|
|
778
|
+
}
|
|
779
|
+
if (typeof value === "string") return `s:${value}`;
|
|
780
|
+
if (typeof value === "boolean") return `b:${value ? "1" : "0"}`;
|
|
781
|
+
throw createError(AGGREGATE_ERROR.FILTER_UNSUPPORTED, `Aggregate min/max only support finite numbers, strings, and booleans. Received '${typeof value}'.`);
|
|
782
|
+
};
|
|
783
|
+
const compareSortableValues = (left, right) => {
|
|
784
|
+
const leftKey = toComparableSortKey(left);
|
|
785
|
+
const rightKey = toComparableSortKey(right);
|
|
786
|
+
if (leftKey < rightKey) return -1;
|
|
787
|
+
if (leftKey > rightKey) return 1;
|
|
788
|
+
return 0;
|
|
789
|
+
};
|
|
790
|
+
const applyExtremaDelta = async (db, tableName, indexName, keyHash, fieldName, value, deltaCount) => {
|
|
791
|
+
if (deltaCount === 0) return;
|
|
792
|
+
const existing = await getExtremaByValue(db, tableName, indexName, keyHash, fieldName, value);
|
|
793
|
+
const now = Date.now();
|
|
794
|
+
if (!existing) {
|
|
795
|
+
if (deltaCount < 0) return;
|
|
796
|
+
await db.insert(AGGREGATE_EXTREMA_TABLE, {
|
|
797
|
+
tableKey: tableName,
|
|
798
|
+
indexName,
|
|
799
|
+
keyHash,
|
|
800
|
+
fieldName,
|
|
801
|
+
valueHash: serializeStable(value),
|
|
802
|
+
value,
|
|
803
|
+
sortKey: toComparableSortKey(value),
|
|
804
|
+
count: deltaCount,
|
|
805
|
+
updatedAt: now
|
|
806
|
+
});
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
const nextCount = existing.count + deltaCount;
|
|
810
|
+
if (nextCount <= 0) {
|
|
811
|
+
await db.delete(AGGREGATE_EXTREMA_TABLE, existing._id);
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
await db.patch(AGGREGATE_EXTREMA_TABLE, existing._id, {
|
|
815
|
+
count: nextCount,
|
|
816
|
+
updatedAt: now
|
|
817
|
+
});
|
|
818
|
+
};
|
|
819
|
+
const buildKeyHashPrefixBounds = (prefixParts) => {
|
|
820
|
+
if (prefixParts.length === 0) return {
|
|
821
|
+
start: "[",
|
|
822
|
+
end: "["
|
|
823
|
+
};
|
|
824
|
+
const start = `${serializeCountKeyParts(prefixParts).slice(0, -1)},`;
|
|
825
|
+
return {
|
|
826
|
+
start,
|
|
827
|
+
end: `${start}\uffff`
|
|
828
|
+
};
|
|
829
|
+
};
|
|
830
|
+
const matchesKeyPrefix = (keyParts, prefixParts) => prefixParts.every((value, index) => deepEquals(keyParts[index], value));
|
|
831
|
+
/**
|
|
832
|
+
* A prefix scan only bounds the key tuple's leading equality fields, so every
|
|
833
|
+
* bucket it returns still has to clear the range predicate and any equality
|
|
834
|
+
* field that sits after the range field in the index.
|
|
835
|
+
*/
|
|
836
|
+
const matchesRangePlanBucket = (options) => {
|
|
837
|
+
const { bucket, indexFields } = options;
|
|
838
|
+
if (!matchesKeyPrefix(bucket.keyParts, options.prefixParts)) return false;
|
|
839
|
+
if (!matchesRangeComparisons(bucket.keyParts[options.rangeFieldIndex], options.comparisons)) return false;
|
|
840
|
+
for (const [field, allowedValues] of Object.entries(options.postFieldSets)) {
|
|
841
|
+
const fieldIndex = indexFields.indexOf(field);
|
|
842
|
+
if (fieldIndex < 0) return false;
|
|
843
|
+
if (!allowedValues.has(serializeStable(bucket.keyParts[fieldIndex]))) return false;
|
|
844
|
+
}
|
|
845
|
+
return true;
|
|
846
|
+
};
|
|
847
|
+
const readPlanBuckets = async (db, plan) => {
|
|
848
|
+
if (!plan.rangeConstraint) return (await mapWithConcurrency(plan.keyCandidates ?? buildCandidateKeys(plan.indexFields, plan.fieldValues), AGGREGATE_BUCKET_READ_CONCURRENCY, (keyParts) => getBucketByKey(db, plan.tableName, plan.indexName, keyParts))).filter((bucket) => bucket !== null);
|
|
849
|
+
const prefixCandidates = buildCandidateKeys(plan.rangeConstraint.prefixFields, plan.fieldValues);
|
|
850
|
+
const rangeFieldIndex = plan.indexFields.indexOf(plan.rangeConstraint.fieldName);
|
|
851
|
+
if (rangeFieldIndex < 0) return [];
|
|
852
|
+
const postFieldSets = Object.fromEntries(Object.entries(plan.postFieldValues).map(([field, values]) => [field, toConstraintSet(values)]));
|
|
853
|
+
const workBudget = plan.rangeConstraint.workBudget ?? DEFAULT_AGGREGATE_WORK_BUDGET;
|
|
854
|
+
const scanLimit = workBudget + 1;
|
|
855
|
+
let pending = prefixCandidates.map((prefixParts) => ({
|
|
856
|
+
prefixParts,
|
|
857
|
+
...buildKeyHashPrefixBounds(prefixParts)
|
|
858
|
+
}));
|
|
859
|
+
const matchedById = /* @__PURE__ */ new Map();
|
|
860
|
+
let scanned = 0;
|
|
861
|
+
while (pending.length > 0 && scanned < scanLimit) {
|
|
862
|
+
const remaining = scanLimit - scanned;
|
|
863
|
+
const batch = pending.slice(0, Math.min(AGGREGATE_BUCKET_READ_CONCURRENCY, remaining));
|
|
864
|
+
const share = Math.floor(remaining / batch.length);
|
|
865
|
+
const pages = await mapWithConcurrency(batch, AGGREGATE_BUCKET_READ_CONCURRENCY, (scan) => listBucketsByHashPrefix(db, plan.tableName, plan.indexName, scan, share));
|
|
866
|
+
const next = pending.slice(batch.length);
|
|
867
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
868
|
+
const scan = batch[index];
|
|
869
|
+
const page = pages[index];
|
|
870
|
+
scanned += page.length;
|
|
871
|
+
for (const bucket of page) if (matchesRangePlanBucket({
|
|
872
|
+
bucket,
|
|
873
|
+
prefixParts: scan.prefixParts,
|
|
874
|
+
indexFields: plan.indexFields,
|
|
875
|
+
rangeFieldIndex,
|
|
876
|
+
comparisons: plan.rangeConstraint.comparisons,
|
|
877
|
+
postFieldSets
|
|
878
|
+
})) matchedById.set(String(bucket._id), bucket);
|
|
879
|
+
const lastBucket = page.at(-1);
|
|
880
|
+
if (page.length < share || !lastBucket) continue;
|
|
881
|
+
next.push({
|
|
882
|
+
...scan,
|
|
883
|
+
after: lastBucket.keyHash
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
pending = next;
|
|
887
|
+
}
|
|
888
|
+
if (scanned > workBudget) {
|
|
889
|
+
const fanOut = prefixCandidates.length > 1 ? ` across ${prefixCandidates.length} equality-prefix combinations` : "";
|
|
890
|
+
throw createError(plan.rangeConstraint.filterErrorCode ?? AGGREGATE_ERROR.FILTER_UNSUPPORTED, `${plan.rangeConstraint.methodName ?? "aggregate()"} range filter on '${plan.rangeConstraint.fieldName}' scans more than ${workBudget} aggregate buckets${fanOut} on aggregateIndex '${plan.indexName}'. Narrow the equality prefix, shrink the IN lists it expands to, reduce the range field's cardinality, add a narrower aggregateIndex, or increase defineSchema(..., { defaults: { aggregateWorkBudget } }).`);
|
|
891
|
+
}
|
|
892
|
+
const buckets = [...matchedById.values()];
|
|
893
|
+
buckets[RANGE_WORK_STATE] = {
|
|
894
|
+
scannedBuckets: scanned,
|
|
895
|
+
reservedExtremaReads: 0
|
|
896
|
+
};
|
|
897
|
+
return buckets;
|
|
898
|
+
};
|
|
899
|
+
const sortFieldValueRecord = (values) => Object.fromEntries(Object.keys(values).sort().map((field) => [field, values[field] ?? []]));
|
|
900
|
+
const getPlanBucketCacheKey = (plan) => serializeStable({
|
|
901
|
+
tableName: plan.tableName,
|
|
902
|
+
indexName: plan.indexName,
|
|
903
|
+
indexFields: plan.indexFields,
|
|
904
|
+
fieldValues: sortFieldValueRecord(plan.fieldValues),
|
|
905
|
+
keyCandidates: plan.keyCandidates ?? null,
|
|
906
|
+
rangeConstraint: plan.rangeConstraint ? {
|
|
907
|
+
fieldName: plan.rangeConstraint.fieldName,
|
|
908
|
+
comparisons: [...plan.rangeConstraint.comparisons],
|
|
909
|
+
prefixFields: plan.rangeConstraint.prefixFields
|
|
910
|
+
} : null,
|
|
911
|
+
postFieldValues: sortFieldValueRecord(plan.postFieldValues)
|
|
912
|
+
});
|
|
913
|
+
const readPlanBucketsWithCache = async (db, plan, bucketCache) => {
|
|
914
|
+
if (!bucketCache) return await readPlanBuckets(db, plan);
|
|
915
|
+
const cacheKey = getPlanBucketCacheKey(plan);
|
|
916
|
+
const existing = bucketCache.get(cacheKey);
|
|
917
|
+
if (existing) return await existing;
|
|
918
|
+
const pending = readPlanBuckets(db, plan);
|
|
919
|
+
bucketCache.set(cacheKey, pending);
|
|
920
|
+
try {
|
|
921
|
+
return await pending;
|
|
922
|
+
} catch (error) {
|
|
923
|
+
bucketCache.delete(cacheKey);
|
|
924
|
+
throw error;
|
|
925
|
+
}
|
|
926
|
+
};
|
|
927
|
+
const readCountFromBuckets = async (db, plan, bucketCache) => {
|
|
928
|
+
return (await readPlanBucketsWithCache(db, plan, bucketCache)).reduce((sum, bucket) => sum + bucket.count, 0);
|
|
929
|
+
};
|
|
930
|
+
const readSumFromBuckets = async (db, plan, bucketCache) => {
|
|
931
|
+
if (plan.metric.kind !== "sum") throw new Error("readSumFromBuckets() requires a sum aggregate plan.");
|
|
932
|
+
let total = 0;
|
|
933
|
+
let totalNonNull = 0;
|
|
934
|
+
const buckets = await readPlanBucketsWithCache(db, plan, bucketCache);
|
|
935
|
+
for (const bucket of buckets) {
|
|
936
|
+
const sumValues = normalizeSumValues(bucket.sumValues);
|
|
937
|
+
const countValues = normalizeNonNullCountValues(bucket.nonNullCountValues);
|
|
938
|
+
total += sumValues[plan.metric.field] ?? 0;
|
|
939
|
+
totalNonNull += countValues[plan.metric.field] ?? 0;
|
|
940
|
+
}
|
|
941
|
+
if (totalNonNull === 0) return null;
|
|
942
|
+
return total;
|
|
943
|
+
};
|
|
944
|
+
const readCountFieldFromBuckets = async (db, plan, bucketCache) => {
|
|
945
|
+
if (plan.metric.kind !== "countField") throw new Error("readCountFieldFromBuckets() requires a countField aggregate plan.");
|
|
946
|
+
let total = 0;
|
|
947
|
+
const buckets = await readPlanBucketsWithCache(db, plan, bucketCache);
|
|
948
|
+
for (const bucket of buckets) {
|
|
949
|
+
const countValues = normalizeNonNullCountValues(bucket.nonNullCountValues);
|
|
950
|
+
total += countValues[plan.metric.field] ?? 0;
|
|
951
|
+
}
|
|
952
|
+
return total;
|
|
953
|
+
};
|
|
954
|
+
const readAverageFromBuckets = async (db, plan, bucketCache) => {
|
|
955
|
+
if (plan.metric.kind !== "avg") throw new Error("readAverageFromBuckets() requires an avg aggregate plan.");
|
|
956
|
+
let totalSum = 0;
|
|
957
|
+
let totalNonNull = 0;
|
|
958
|
+
const buckets = await readPlanBucketsWithCache(db, plan, bucketCache);
|
|
959
|
+
for (const bucket of buckets) {
|
|
960
|
+
const sumValues = normalizeSumValues(bucket.sumValues);
|
|
961
|
+
const countValues = normalizeNonNullCountValues(bucket.nonNullCountValues);
|
|
962
|
+
totalSum += sumValues[plan.metric.field] ?? 0;
|
|
963
|
+
totalNonNull += countValues[plan.metric.field] ?? 0;
|
|
964
|
+
}
|
|
965
|
+
if (totalNonNull === 0) return null;
|
|
966
|
+
return totalSum / totalNonNull;
|
|
967
|
+
};
|
|
968
|
+
const readKeyExtrema = async (db, params) => {
|
|
969
|
+
let query = db.query(AGGREGATE_EXTREMA_TABLE).withIndex("by_table_index_hash_field_sort", (q) => q.eq("tableKey", params.tableName).eq("indexName", params.indexName).eq("keyHash", params.keyHash).eq("fieldName", params.fieldName));
|
|
970
|
+
if (params.kind === "max") query = query.order("desc");
|
|
971
|
+
return (await query.take(1))[0]?.value ?? null;
|
|
972
|
+
};
|
|
973
|
+
const readExtremaFromBuckets = async (db, plan, bucketCache) => {
|
|
974
|
+
if (plan.metric.kind !== "min" && plan.metric.kind !== "max") throw new Error("readExtremaFromBuckets() requires a min/max aggregate plan.");
|
|
975
|
+
let selected = null;
|
|
976
|
+
const buckets = await readPlanBucketsWithCache(db, plan, bucketCache);
|
|
977
|
+
const metric = plan.metric;
|
|
978
|
+
const rangeWork = buckets[RANGE_WORK_STATE];
|
|
979
|
+
if (rangeWork && plan.rangeConstraint) {
|
|
980
|
+
const workBudget = plan.rangeConstraint.workBudget ?? DEFAULT_AGGREGATE_WORK_BUDGET;
|
|
981
|
+
const requestedWork = buckets.length;
|
|
982
|
+
const totalWork = rangeWork.scannedBuckets + rangeWork.reservedExtremaReads + requestedWork;
|
|
983
|
+
if (totalWork > workBudget) throw createError(plan.rangeConstraint.filterErrorCode ?? AGGREGATE_ERROR.FILTER_UNSUPPORTED, `${plan.rangeConstraint.methodName ?? "aggregate()"} range filter on '${plan.rangeConstraint.fieldName}' needs ${totalWork} aggregate work units (${rangeWork.scannedBuckets} bucket reads and ${rangeWork.reservedExtremaReads + requestedWork} extrema reads), exceeding aggregateWorkBudget (${workBudget}) on aggregateIndex '${plan.indexName}'. Narrow the range, add a narrower aggregateIndex, or increase defineSchema(..., { defaults: { aggregateWorkBudget } }).`);
|
|
984
|
+
rangeWork.reservedExtremaReads += requestedWork;
|
|
985
|
+
}
|
|
986
|
+
const values = await mapWithConcurrency(buckets, AGGREGATE_BUCKET_READ_CONCURRENCY, (bucket) => readKeyExtrema(db, {
|
|
987
|
+
tableName: plan.tableName,
|
|
988
|
+
indexName: plan.indexName,
|
|
989
|
+
keyHash: bucket.keyHash,
|
|
990
|
+
fieldName: metric.field,
|
|
991
|
+
kind: metric.kind
|
|
992
|
+
}));
|
|
993
|
+
for (const value of values) {
|
|
994
|
+
if (value === null || value === void 0) continue;
|
|
995
|
+
if (selected === null) {
|
|
996
|
+
selected = value;
|
|
997
|
+
continue;
|
|
998
|
+
}
|
|
999
|
+
const comparison = compareSortableValues(value, selected);
|
|
1000
|
+
if (plan.metric.kind === "min" && comparison < 0) {
|
|
1001
|
+
selected = value;
|
|
1002
|
+
continue;
|
|
1003
|
+
}
|
|
1004
|
+
if (plan.metric.kind === "max" && comparison > 0) selected = value;
|
|
1005
|
+
}
|
|
1006
|
+
return selected;
|
|
1007
|
+
};
|
|
1008
|
+
const getMemberByDoc = async (db, tableName, indexName, docId) => {
|
|
1009
|
+
return (await db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index_doc", (q) => q.eq("kind", AGGREGATE_STATE_KIND_METRIC).eq("tableKey", tableName).eq("indexName", indexName).eq("docId", docId)).collect())[0] ?? null;
|
|
1010
|
+
};
|
|
1011
|
+
const isComparableAggregateValue = (value) => {
|
|
1012
|
+
if (typeof value === "number") return Number.isFinite(value);
|
|
1013
|
+
return typeof value === "string" || typeof value === "boolean";
|
|
1014
|
+
};
|
|
1015
|
+
const computeAggregateMetricValues = (doc, definition) => {
|
|
1016
|
+
const sumValues = {};
|
|
1017
|
+
for (const field of [...definition.sumFields, ...definition.avgFields]) {
|
|
1018
|
+
const value = doc[field];
|
|
1019
|
+
if (value === null || value === void 0) continue;
|
|
1020
|
+
if (typeof value !== "number" || !Number.isFinite(value)) throw createError(AGGREGATE_ERROR.FILTER_UNSUPPORTED, `aggregateIndex '${definition.name}' sum('${field}') requires finite number values.`);
|
|
1021
|
+
sumValues[field] = value;
|
|
1022
|
+
}
|
|
1023
|
+
const nonNullCountValues = {};
|
|
1024
|
+
for (const field of [
|
|
1025
|
+
...definition.countFields,
|
|
1026
|
+
...definition.sumFields,
|
|
1027
|
+
...definition.avgFields
|
|
1028
|
+
]) {
|
|
1029
|
+
const value = doc[field];
|
|
1030
|
+
if (value === null || value === void 0) continue;
|
|
1031
|
+
nonNullCountValues[field] = 1;
|
|
1032
|
+
}
|
|
1033
|
+
const extremaFields = new Set([...definition.minFields, ...definition.maxFields]);
|
|
1034
|
+
const extremaValues = {};
|
|
1035
|
+
for (const field of extremaFields) {
|
|
1036
|
+
const value = doc[field];
|
|
1037
|
+
if (value === null || value === void 0) continue;
|
|
1038
|
+
if (!isComparableAggregateValue(value)) throw createError(AGGREGATE_ERROR.FILTER_UNSUPPORTED, `aggregateIndex '${definition.name}' ${field} must be a finite number, string, or boolean for min/max.`);
|
|
1039
|
+
extremaValues[field] = value;
|
|
1040
|
+
}
|
|
1041
|
+
return {
|
|
1042
|
+
sumValues,
|
|
1043
|
+
extremaValues,
|
|
1044
|
+
nonNullCountValues
|
|
1045
|
+
};
|
|
1046
|
+
};
|
|
1047
|
+
const toExtremaDeltas = (keyHash, values, delta) => Object.entries(values).map(([fieldName, value]) => ({
|
|
1048
|
+
keyHash,
|
|
1049
|
+
fieldName,
|
|
1050
|
+
value,
|
|
1051
|
+
delta
|
|
1052
|
+
}));
|
|
1053
|
+
const accumulateNumberValues = (target, source) => {
|
|
1054
|
+
for (const [field, value] of Object.entries(source)) {
|
|
1055
|
+
const next = (target[field] ?? 0) + value;
|
|
1056
|
+
if (next === 0) {
|
|
1057
|
+
delete target[field];
|
|
1058
|
+
continue;
|
|
1059
|
+
}
|
|
1060
|
+
target[field] = next;
|
|
1061
|
+
}
|
|
1062
|
+
};
|
|
1063
|
+
/**
|
|
1064
|
+
* Pure half of aggregate reconciliation: works out what a single document's
|
|
1065
|
+
* membership change implies, without touching the database. Callers batch these
|
|
1066
|
+
* and flush once so a page of documents sharing a key tuple writes its bucket
|
|
1067
|
+
* one time instead of once per document.
|
|
1068
|
+
*/
|
|
1069
|
+
const computeMembershipDelta = (existing, params) => {
|
|
1070
|
+
const { tableName, indexName, docId, keyParts, metricValues } = params;
|
|
1071
|
+
if (!keyParts || !metricValues) {
|
|
1072
|
+
if (!existing) return {
|
|
1073
|
+
buckets: [],
|
|
1074
|
+
extrema: [],
|
|
1075
|
+
member: { kind: "none" }
|
|
1076
|
+
};
|
|
1077
|
+
return {
|
|
1078
|
+
buckets: [{
|
|
1079
|
+
keyHash: serializeCountKeyParts(existing.keyParts),
|
|
1080
|
+
keyParts: existing.keyParts,
|
|
1081
|
+
deltaCount: -1,
|
|
1082
|
+
deltaSums: negateSumValues(normalizeSumValues(existing.sumValues)),
|
|
1083
|
+
deltaNonNullCounts: negateCountValues(normalizeNonNullCountValues(existing.nonNullCountValues))
|
|
1084
|
+
}],
|
|
1085
|
+
extrema: toExtremaDeltas(existing.keyHash, normalizeExtremaValues(existing.extremaValues), -1),
|
|
1086
|
+
member: {
|
|
1087
|
+
kind: "delete",
|
|
1088
|
+
id: existing._id
|
|
1089
|
+
}
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
const normalizedKeyParts = keyParts.map((part) => normalizeUndefined(part));
|
|
1093
|
+
const keyHash = serializeCountKeyParts(normalizedKeyParts);
|
|
1094
|
+
const now = Date.now();
|
|
1095
|
+
const normalizedNextSumValues = normalizeSumValues(metricValues.sumValues);
|
|
1096
|
+
const normalizedNextNonNullCountValues = normalizeNonNullCountValues(metricValues.nonNullCountValues);
|
|
1097
|
+
const normalizedNextExtremaValues = normalizeExtremaValues(metricValues.extremaValues);
|
|
1098
|
+
if (existing && existing.keyHash === keyHash && deepEquals(existing.keyParts, normalizedKeyParts) && deepEquals(normalizeSumValues(existing.sumValues), normalizedNextSumValues) && deepEquals(normalizeNonNullCountValues(existing.nonNullCountValues), normalizedNextNonNullCountValues) && deepEquals(normalizeExtremaValues(existing.extremaValues), normalizedNextExtremaValues)) return {
|
|
1099
|
+
buckets: [],
|
|
1100
|
+
extrema: [],
|
|
1101
|
+
member: { kind: "none" }
|
|
1102
|
+
};
|
|
1103
|
+
const buckets = [];
|
|
1104
|
+
const extrema = [];
|
|
1105
|
+
if (existing) {
|
|
1106
|
+
buckets.push({
|
|
1107
|
+
keyHash: serializeCountKeyParts(existing.keyParts),
|
|
1108
|
+
keyParts: existing.keyParts,
|
|
1109
|
+
deltaCount: -1,
|
|
1110
|
+
deltaSums: negateSumValues(normalizeSumValues(existing.sumValues)),
|
|
1111
|
+
deltaNonNullCounts: negateCountValues(normalizeNonNullCountValues(existing.nonNullCountValues))
|
|
1112
|
+
});
|
|
1113
|
+
extrema.push(...toExtremaDeltas(existing.keyHash, normalizeExtremaValues(existing.extremaValues), -1));
|
|
1114
|
+
}
|
|
1115
|
+
buckets.push({
|
|
1116
|
+
keyHash,
|
|
1117
|
+
keyParts: normalizedKeyParts,
|
|
1118
|
+
deltaCount: 1,
|
|
1119
|
+
deltaSums: normalizedNextSumValues,
|
|
1120
|
+
deltaNonNullCounts: normalizedNextNonNullCountValues
|
|
1121
|
+
});
|
|
1122
|
+
extrema.push(...toExtremaDeltas(keyHash, normalizedNextExtremaValues, 1));
|
|
1123
|
+
const memberFields = {
|
|
1124
|
+
kind: AGGREGATE_STATE_KIND_METRIC,
|
|
1125
|
+
keyHash,
|
|
1126
|
+
keyParts: normalizedKeyParts,
|
|
1127
|
+
sumValues: normalizedNextSumValues,
|
|
1128
|
+
nonNullCountValues: normalizedNextNonNullCountValues,
|
|
1129
|
+
extremaValues: normalizedNextExtremaValues,
|
|
1130
|
+
updatedAt: now
|
|
1131
|
+
};
|
|
1132
|
+
return {
|
|
1133
|
+
buckets,
|
|
1134
|
+
extrema,
|
|
1135
|
+
member: existing ? {
|
|
1136
|
+
kind: "patch",
|
|
1137
|
+
id: existing._id,
|
|
1138
|
+
doc: memberFields
|
|
1139
|
+
} : {
|
|
1140
|
+
kind: "insert",
|
|
1141
|
+
doc: {
|
|
1142
|
+
...memberFields,
|
|
1143
|
+
tableKey: tableName,
|
|
1144
|
+
indexName,
|
|
1145
|
+
docId
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
};
|
|
1149
|
+
};
|
|
1150
|
+
const computeAggregateMembershipDelta = async (db, params) => {
|
|
1151
|
+
return computeMembershipDelta(await getMemberByDoc(db, params.tableName, params.indexName, params.docId), params);
|
|
1152
|
+
};
|
|
1153
|
+
/**
|
|
1154
|
+
* Write half of aggregate reconciliation. Folds bucket deltas by key tuple and
|
|
1155
|
+
* extrema deltas by (keyHash, field, value) so each storage document is read and
|
|
1156
|
+
* written once regardless of how many source documents contributed to it.
|
|
1157
|
+
*/
|
|
1158
|
+
const flushAggregateMembershipDeltas = async (db, tableName, indexName, deltas) => {
|
|
1159
|
+
const bucketDeltas = /* @__PURE__ */ new Map();
|
|
1160
|
+
const extremaDeltas = /* @__PURE__ */ new Map();
|
|
1161
|
+
for (const delta of deltas) {
|
|
1162
|
+
for (const bucket of delta.buckets) {
|
|
1163
|
+
const current = bucketDeltas.get(bucket.keyHash);
|
|
1164
|
+
if (!current) {
|
|
1165
|
+
bucketDeltas.set(bucket.keyHash, {
|
|
1166
|
+
keyHash: bucket.keyHash,
|
|
1167
|
+
keyParts: bucket.keyParts,
|
|
1168
|
+
deltaCount: bucket.deltaCount,
|
|
1169
|
+
deltaSums: { ...bucket.deltaSums },
|
|
1170
|
+
deltaNonNullCounts: { ...bucket.deltaNonNullCounts }
|
|
1171
|
+
});
|
|
1172
|
+
continue;
|
|
1173
|
+
}
|
|
1174
|
+
current.deltaCount += bucket.deltaCount;
|
|
1175
|
+
accumulateNumberValues(current.deltaSums, bucket.deltaSums);
|
|
1176
|
+
accumulateNumberValues(current.deltaNonNullCounts, bucket.deltaNonNullCounts);
|
|
1177
|
+
}
|
|
1178
|
+
for (const entry of delta.extrema) {
|
|
1179
|
+
const cacheKey = serializeStable([
|
|
1180
|
+
entry.keyHash,
|
|
1181
|
+
entry.fieldName,
|
|
1182
|
+
serializeStable(entry.value)
|
|
1183
|
+
]);
|
|
1184
|
+
const current = extremaDeltas.get(cacheKey);
|
|
1185
|
+
if (!current) {
|
|
1186
|
+
extremaDeltas.set(cacheKey, { ...entry });
|
|
1187
|
+
continue;
|
|
1188
|
+
}
|
|
1189
|
+
current.delta += entry.delta;
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
for (const bucket of bucketDeltas.values()) await applyBucketDelta(db, tableName, indexName, bucket.keyParts, bucket.deltaCount, bucket.deltaSums, bucket.deltaNonNullCounts);
|
|
1193
|
+
for (const entry of extremaDeltas.values()) await applyExtremaDelta(db, tableName, indexName, entry.keyHash, entry.fieldName, entry.value, entry.delta);
|
|
1194
|
+
for (const delta of deltas) {
|
|
1195
|
+
const member = delta.member;
|
|
1196
|
+
if (member.kind === "delete") {
|
|
1197
|
+
await db.delete(AGGREGATE_MEMBER_TABLE, member.id);
|
|
1198
|
+
continue;
|
|
1199
|
+
}
|
|
1200
|
+
if (member.kind === "patch") {
|
|
1201
|
+
await db.patch(AGGREGATE_MEMBER_TABLE, member.id, member.doc);
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
if (member.kind === "insert") await db.insert(AGGREGATE_MEMBER_TABLE, member.doc);
|
|
1205
|
+
}
|
|
1206
|
+
};
|
|
1207
|
+
/**
|
|
1208
|
+
* Single-document reconciliation. Flushes eagerly so user code reading an
|
|
1209
|
+
* aggregate later in the same mutation sees its own writes.
|
|
1210
|
+
*/
|
|
1211
|
+
const reconcileAggregateMembership = async (db, params) => {
|
|
1212
|
+
const delta = await computeAggregateMembershipDelta(db, params);
|
|
1213
|
+
await flushAggregateMembershipDeltas(db, params.tableName, params.indexName, [delta]);
|
|
1214
|
+
};
|
|
1215
|
+
const computeCountKeyParts = (doc, fields) => fields.map((field) => normalizeUndefined(doc[field]));
|
|
1216
|
+
const applyAggregateIndexesForChange = async (db, tableName, aggregateIndexes, change) => {
|
|
1217
|
+
const docId = String(change.id);
|
|
1218
|
+
for (const aggregateIndex of aggregateIndexes) {
|
|
1219
|
+
const keyParts = change.operation === "delete" ? null : computeCountKeyParts(change.newDoc, aggregateIndex.fields);
|
|
1220
|
+
const metricValues = change.operation === "delete" ? null : computeAggregateMetricValues(change.newDoc, aggregateIndex);
|
|
1221
|
+
await reconcileAggregateMembership(db, {
|
|
1222
|
+
tableName,
|
|
1223
|
+
indexName: aggregateIndex.name,
|
|
1224
|
+
docId,
|
|
1225
|
+
keyParts,
|
|
1226
|
+
metricValues
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
};
|
|
1230
|
+
const getCountState = async (db, tableName, indexName, kind = AGGREGATE_STATE_KIND_METRIC) => {
|
|
1231
|
+
const state = (await db.query(AGGREGATE_STATE_TABLE).withIndex("by_kind_table_index", (q) => q.eq("kind", kind).eq("tableKey", tableName).eq("indexName", indexName)).collect())[0];
|
|
1232
|
+
if (!state) return null;
|
|
1233
|
+
const { tableKey, ...rest } = state;
|
|
1234
|
+
return {
|
|
1235
|
+
...rest,
|
|
1236
|
+
tableName: tableKey
|
|
1237
|
+
};
|
|
1238
|
+
};
|
|
1239
|
+
const assertAggregateIndexesWritable = async (db, tableName, metricIndexNames, rankIndexNames) => {
|
|
1240
|
+
const clearingStates = await db.query(AGGREGATE_STATE_TABLE).withIndex("by_table_status", (q) => q.eq("tableKey", tableName).eq("status", COUNT_STATUS_CLEARING)).collect();
|
|
1241
|
+
const metricNames = new Set(metricIndexNames);
|
|
1242
|
+
const rankNames = new Set(rankIndexNames);
|
|
1243
|
+
const blockingState = clearingStates.find((state) => state.kind === AGGREGATE_STATE_KIND_RANK ? rankNames.has(state.indexName) : metricNames.has(state.indexName));
|
|
1244
|
+
if (!blockingState) return;
|
|
1245
|
+
const indexType = blockingState.kind === AGGREGATE_STATE_KIND_RANK ? "rankIndex" : "aggregateIndex";
|
|
1246
|
+
throw createError(blockingState.kind === AGGREGATE_STATE_KIND_RANK ? "RANK_INDEX_BUILDING" : AGGREGATE_ERROR.INDEX_BUILDING, `${indexType} '${tableName}.${blockingState.indexName}' is CLEARING. Retry the write after aggregateBackfill reaches BUILDING or READY.`);
|
|
1247
|
+
};
|
|
1248
|
+
const setCountState = async (db, nextState, kind = AGGREGATE_STATE_KIND_METRIC) => {
|
|
1249
|
+
const existing = await getCountState(db, nextState.tableName, nextState.indexName, kind);
|
|
1250
|
+
const payload = {
|
|
1251
|
+
kind,
|
|
1252
|
+
tableKey: nextState.tableName,
|
|
1253
|
+
indexName: nextState.indexName,
|
|
1254
|
+
keyDefinitionHash: nextState.keyDefinitionHash,
|
|
1255
|
+
metricDefinitionHash: nextState.metricDefinitionHash,
|
|
1256
|
+
status: nextState.status,
|
|
1257
|
+
cursor: nextState.cursor ?? null,
|
|
1258
|
+
processed: nextState.processed,
|
|
1259
|
+
startedAt: nextState.startedAt,
|
|
1260
|
+
updatedAt: nextState.updatedAt,
|
|
1261
|
+
completedAt: nextState.completedAt ?? null,
|
|
1262
|
+
lastError: nextState.lastError ?? null
|
|
1263
|
+
};
|
|
1264
|
+
if (!existing) {
|
|
1265
|
+
await db.insert(AGGREGATE_STATE_TABLE, payload);
|
|
1266
|
+
return;
|
|
1267
|
+
}
|
|
1268
|
+
await db.patch(AGGREGATE_STATE_TABLE, existing._id, payload);
|
|
1269
|
+
};
|
|
1270
|
+
const setCountStateError = async (db, tableName, indexName, error, kind = AGGREGATE_STATE_KIND_METRIC) => {
|
|
1271
|
+
const now = Date.now();
|
|
1272
|
+
const existing = await getCountState(db, tableName, indexName, kind);
|
|
1273
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1274
|
+
if (!existing) throw new Error(`Missing count state for '${tableName}.${indexName}' while recording backfill error.`);
|
|
1275
|
+
await db.patch(AGGREGATE_STATE_TABLE, existing._id, {
|
|
1276
|
+
status: existing.status === COUNT_STATUS_CLEARING ? COUNT_STATUS_CLEARING : COUNT_STATUS_BUILDING,
|
|
1277
|
+
updatedAt: now,
|
|
1278
|
+
keyDefinitionHash: existing.keyDefinitionHash,
|
|
1279
|
+
metricDefinitionHash: existing.metricDefinitionHash,
|
|
1280
|
+
lastError: message
|
|
1281
|
+
});
|
|
1282
|
+
};
|
|
1283
|
+
/**
|
|
1284
|
+
* Removes at most `batchSize` documents of an aggregate index's stored state and
|
|
1285
|
+
* reports whether anything is left. Callers drive it to completion across
|
|
1286
|
+
* transactions, so clearing a large index never has to fit in one mutation.
|
|
1287
|
+
*
|
|
1288
|
+
* Members are removed through the normal delta machinery rather than raw
|
|
1289
|
+
* deletes, so buckets and extrema stay consistent with the members that remain
|
|
1290
|
+
* at every intermediate step. That keeps concurrent writers correct while the
|
|
1291
|
+
* clear drains. Residual bucket/extrema rows (drift with no member behind them)
|
|
1292
|
+
* are swept only once no members are left, and the loop re-checks members
|
|
1293
|
+
* afterwards.
|
|
1294
|
+
*/
|
|
1295
|
+
const clearCountIndexChunk = async (db, tableName, indexName, batchSize) => {
|
|
1296
|
+
const members = await takeMembersForIndex(db, tableName, indexName, batchSize);
|
|
1297
|
+
if (members.length > 0) {
|
|
1298
|
+
await flushAggregateMembershipDeltas(db, tableName, indexName, members.map((member) => computeMembershipDelta(member, {
|
|
1299
|
+
tableName,
|
|
1300
|
+
indexName,
|
|
1301
|
+
docId: member.docId,
|
|
1302
|
+
keyParts: null,
|
|
1303
|
+
metricValues: null
|
|
1304
|
+
})));
|
|
1305
|
+
return {
|
|
1306
|
+
done: false,
|
|
1307
|
+
processed: members.length
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
const buckets = await takeBucketsForIndex(db, tableName, indexName, batchSize);
|
|
1311
|
+
if (buckets.length > 0) {
|
|
1312
|
+
for (const bucket of buckets) await db.delete(AGGREGATE_BUCKET_TABLE, bucket._id);
|
|
1313
|
+
return {
|
|
1314
|
+
done: false,
|
|
1315
|
+
processed: buckets.length
|
|
1316
|
+
};
|
|
1317
|
+
}
|
|
1318
|
+
const extrema = await takeExtremaForIndex(db, tableName, indexName, batchSize);
|
|
1319
|
+
if (extrema.length > 0) {
|
|
1320
|
+
for (const entry of extrema) await db.delete(AGGREGATE_EXTREMA_TABLE, entry._id);
|
|
1321
|
+
return {
|
|
1322
|
+
done: false,
|
|
1323
|
+
processed: extrema.length
|
|
1324
|
+
};
|
|
1325
|
+
}
|
|
1326
|
+
return {
|
|
1327
|
+
done: true,
|
|
1328
|
+
processed: 0
|
|
1329
|
+
};
|
|
1330
|
+
};
|
|
1331
|
+
const ensureIndexReady = async (db, tableName, indexName, code, indexLabel) => {
|
|
1332
|
+
const state = await getCountState(db, tableName, indexName);
|
|
1333
|
+
if (!state || state.status !== COUNT_STATUS_READY) throw createError(code, `${indexLabel} '${tableName}.${indexName}' is BUILDING. Run aggregateBackfill until READY.`);
|
|
1334
|
+
};
|
|
1335
|
+
const ensureCountIndexReady = async (db, tableName, indexName) => ensureIndexReady(db, tableName, indexName, COUNT_ERROR.INDEX_BUILDING, "aggregateIndex");
|
|
1336
|
+
const ensureAggregateIndexReady = async (db, tableName, indexName) => ensureIndexReady(db, tableName, indexName, AGGREGATE_ERROR.INDEX_BUILDING, "aggregateIndex");
|
|
1337
|
+
const listSchemaAggregateIndexes = (schema) => {
|
|
1338
|
+
const entries = [];
|
|
1339
|
+
for (const tableConfig of Object.values(schema)) {
|
|
1340
|
+
const aggregateIndexes = getAggregateIndexDefinitions(tableConfig);
|
|
1341
|
+
for (const aggregateIndex of aggregateIndexes) entries.push({
|
|
1342
|
+
tableName: tableConfig.name,
|
|
1343
|
+
indexName: aggregateIndex.name,
|
|
1344
|
+
fields: aggregateIndex.fields,
|
|
1345
|
+
countFields: aggregateIndex.countFields,
|
|
1346
|
+
sumFields: aggregateIndex.sumFields,
|
|
1347
|
+
avgFields: aggregateIndex.avgFields,
|
|
1348
|
+
minFields: aggregateIndex.minFields,
|
|
1349
|
+
maxFields: aggregateIndex.maxFields
|
|
1350
|
+
});
|
|
1351
|
+
}
|
|
1352
|
+
return entries;
|
|
1353
|
+
};
|
|
1354
|
+
const isIndexCountZero = (plan) => Array.isArray(plan.keyCandidates) && plan.keyCandidates.length === 0 || [...Object.values(plan.fieldValues), ...Object.values(plan.postFieldValues)].some((values) => values.length === 0);
|
|
1355
|
+
const isAggregatePlanZero = (plan) => Array.isArray(plan.keyCandidates) && plan.keyCandidates.length === 0 || [...Object.values(plan.fieldValues), ...Object.values(plan.postFieldValues)].some((values) => values.length === 0);
|
|
1356
|
+
|
|
1357
|
+
//#endregion
|
|
1358
|
+
//#region src/orm/aggregate-index/rank-runtime.ts
|
|
1359
|
+
const isObject = (value) => !!value && typeof value === "object" && !Array.isArray(value);
|
|
1360
|
+
const stableEquals = (left, right) => JSON.stringify(left) === JSON.stringify(right);
|
|
1361
|
+
const RANK_MEMBER_KIND = AGGREGATE_STATE_KIND_RANK;
|
|
1362
|
+
const RANK_ERROR = {
|
|
1363
|
+
FILTER_UNSUPPORTED: "RANK_FILTER_UNSUPPORTED",
|
|
1364
|
+
NOT_INDEXED: "RANK_NOT_INDEXED",
|
|
1365
|
+
INDEX_BUILDING: "RANK_INDEX_BUILDING",
|
|
1366
|
+
RLS_UNSUPPORTED: "RANK_RLS_UNSUPPORTED"
|
|
1367
|
+
};
|
|
1368
|
+
const createRankError = (code, message) => /* @__PURE__ */ new Error(`${code}: ${message}`);
|
|
1369
|
+
const rankAggregateName = (tableName, indexName) => `${tableName}.${indexName}`;
|
|
1370
|
+
const rankAggregate = (tableName, indexName) => new DirectAggregate({ name: rankAggregateName(tableName, indexName) });
|
|
1371
|
+
const toRankValue = (value) => {
|
|
1372
|
+
if (value === null || value === void 0) return null;
|
|
1373
|
+
if (value instanceof Date) return value.getTime();
|
|
1374
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
1375
|
+
throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, `rankIndex order field values must be finite numbers or dates; received '${typeof value}'.`);
|
|
1376
|
+
};
|
|
1377
|
+
const decodeRankValue = (value, direction) => {
|
|
1378
|
+
if (value === null || value === void 0) return null;
|
|
1379
|
+
if (typeof value === "number") return direction === "desc" ? -value : value;
|
|
1380
|
+
return value;
|
|
1381
|
+
};
|
|
1382
|
+
const getRankSortKey = (definition, doc, docId) => {
|
|
1383
|
+
const key = [];
|
|
1384
|
+
for (const orderField of definition.orderFields) {
|
|
1385
|
+
const rawValue = toRankValue(doc[orderField.field]);
|
|
1386
|
+
if (rawValue === null) {
|
|
1387
|
+
key.push(null);
|
|
1388
|
+
continue;
|
|
1389
|
+
}
|
|
1390
|
+
key.push(orderField.direction === "desc" ? -rawValue : rawValue);
|
|
1391
|
+
}
|
|
1392
|
+
key.push(docId);
|
|
1393
|
+
return key;
|
|
1394
|
+
};
|
|
1395
|
+
const toPublicRankKey = (definition, key) => {
|
|
1396
|
+
if (!Array.isArray(key)) return key;
|
|
1397
|
+
const restored = key.slice(0, definition.orderFields.length).map((value, index) => decodeRankValue(value, definition.orderFields[index].direction));
|
|
1398
|
+
return restored.length === 1 ? restored[0] : restored;
|
|
1399
|
+
};
|
|
1400
|
+
const getRankSumValue = (definition, doc) => {
|
|
1401
|
+
if (!definition.sumField) return 1;
|
|
1402
|
+
const raw = doc[definition.sumField];
|
|
1403
|
+
if (raw === null || raw === void 0) return 0;
|
|
1404
|
+
if (typeof raw !== "number" || !Number.isFinite(raw)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, `rankIndex '${definition.name}' sum field '${definition.sumField}' must be a finite number.`);
|
|
1405
|
+
return raw;
|
|
1406
|
+
};
|
|
1407
|
+
const validateRankWhereValue = (value) => {
|
|
1408
|
+
if (value === null || value === void 0) return null;
|
|
1409
|
+
if (isObject(value)) {
|
|
1410
|
+
if (Object.hasOwn(value, "eq")) return value.eq ?? null;
|
|
1411
|
+
if (value.isNull === true) return null;
|
|
1412
|
+
throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where supports only eq and isNull: true on partition fields.");
|
|
1413
|
+
}
|
|
1414
|
+
if (Array.isArray(value)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where does not support IN arrays in v1.");
|
|
1415
|
+
return value;
|
|
1416
|
+
};
|
|
1417
|
+
const collectWhere = (where, fields, out) => {
|
|
1418
|
+
if (Object.hasOwn(where, "OR") || Object.hasOwn(where, "NOT")) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where supports conjunctions only (object fields + AND).");
|
|
1419
|
+
if (Object.hasOwn(where, "RAW")) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where does not support RAW filters.");
|
|
1420
|
+
const andEntries = where.AND;
|
|
1421
|
+
if (andEntries !== void 0) {
|
|
1422
|
+
if (!Array.isArray(andEntries)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where AND must be an array.");
|
|
1423
|
+
for (const entry of andEntries) {
|
|
1424
|
+
if (!isObject(entry)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where AND entries must be objects.");
|
|
1425
|
+
collectWhere(entry, fields, out);
|
|
1426
|
+
}
|
|
1427
|
+
}
|
|
1428
|
+
for (const [field, rawValue] of Object.entries(where)) {
|
|
1429
|
+
if (field === "AND") continue;
|
|
1430
|
+
if (!fields.has(field)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, `rank() where field '${field}' is not a partition field.`);
|
|
1431
|
+
const nextValue = validateRankWhereValue(rawValue);
|
|
1432
|
+
const prevValue = out.get(field);
|
|
1433
|
+
if (prevValue !== void 0 && !stableEquals(prevValue, nextValue)) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, `rank() where has conflicting constraints for '${field}'.`);
|
|
1434
|
+
out.set(field, nextValue);
|
|
1435
|
+
}
|
|
1436
|
+
};
|
|
1437
|
+
const buildNamespace = (definition, values) => {
|
|
1438
|
+
if (definition.partitionFields.length === 0) return;
|
|
1439
|
+
const parts = definition.partitionFields.map((field) => values.get(field));
|
|
1440
|
+
return parts.length === 1 ? parts[0] : parts;
|
|
1441
|
+
};
|
|
1442
|
+
const ensureRankAllowedForRls = (tableConfig, rlsMode) => {
|
|
1443
|
+
if ((typeof tableConfig.table.isRlsEnabled === "function" ? tableConfig.table.isRlsEnabled() : false) && rlsMode !== "skip") throw createRankError(RANK_ERROR.RLS_UNSUPPORTED, `rank() is not available for table '${tableConfig.name}' in RLS-restricted contexts in v1.`);
|
|
1444
|
+
};
|
|
1445
|
+
const compileRankPlan = (tableConfig, indexName, where) => {
|
|
1446
|
+
const definition = getRankIndexDefinitions(tableConfig).find((entry) => entry.name === indexName);
|
|
1447
|
+
if (!definition) throw createRankError(RANK_ERROR.NOT_INDEXED, `No rankIndex '${indexName}' found on '${tableConfig.name}'. Declare rankIndex('${indexName}') and run aggregateBackfill.`);
|
|
1448
|
+
if (!isObject(where ?? {})) throw createRankError(RANK_ERROR.FILTER_UNSUPPORTED, "rank() where must be an object filter.");
|
|
1449
|
+
const values = /* @__PURE__ */ new Map();
|
|
1450
|
+
collectWhere(where ?? {}, new Set(definition.partitionFields), values);
|
|
1451
|
+
for (const field of definition.partitionFields) if (!values.has(field)) throw createRankError(RANK_ERROR.NOT_INDEXED, `rank() on '${tableConfig.name}.${indexName}' requires all partition fields constrained: [${definition.partitionFields.join(", ")}].`);
|
|
1452
|
+
return {
|
|
1453
|
+
tableName: tableConfig.name,
|
|
1454
|
+
indexName,
|
|
1455
|
+
definition,
|
|
1456
|
+
namespace: buildNamespace(definition, values)
|
|
1457
|
+
};
|
|
1458
|
+
};
|
|
1459
|
+
const ensureRankIndexReady = async (db, tableName, indexName) => {
|
|
1460
|
+
const state = await getCountState(db, tableName, indexName, AGGREGATE_STATE_KIND_RANK);
|
|
1461
|
+
if (!state || state.status !== COUNT_STATUS_READY) throw createRankError(RANK_ERROR.INDEX_BUILDING, `rankIndex '${tableName}.${indexName}' is BUILDING. Run aggregateBackfill until READY.`);
|
|
1462
|
+
};
|
|
1463
|
+
const getRankMemberByDoc = async (db, tableName, indexName, docId) => {
|
|
1464
|
+
return (await db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index_doc", (q) => q.eq("kind", RANK_MEMBER_KIND).eq("tableKey", tableName).eq("indexName", indexName).eq("docId", docId)).collect())[0] ?? null;
|
|
1465
|
+
};
|
|
1466
|
+
const takeRankMembers = async (db, tableName, indexName, limit) => await db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index", (q) => q.eq("kind", RANK_MEMBER_KIND).eq("tableKey", tableName).eq("indexName", indexName)).take(limit);
|
|
1467
|
+
const rankCtx = (db) => ({
|
|
1468
|
+
db,
|
|
1469
|
+
orm: void 0
|
|
1470
|
+
});
|
|
1471
|
+
/** Trees dropped per invocation once every rank member has been removed. */
|
|
1472
|
+
const RANK_TREE_DROP_BATCH = 16;
|
|
1473
|
+
/**
|
|
1474
|
+
* Removes at most `batchSize` rank members and reports whether anything is
|
|
1475
|
+
* left. Each member is removed from the btree before its row is dropped, so the
|
|
1476
|
+
* tree stays consistent with the members that remain and a partially drained
|
|
1477
|
+
* clear can safely resume in a later mutation.
|
|
1478
|
+
*/
|
|
1479
|
+
const clearRankIndexChunk = async (db, tableName, indexName, batchSize) => {
|
|
1480
|
+
const aggregate = rankAggregate(tableName, indexName);
|
|
1481
|
+
const ctx = rankCtx(db);
|
|
1482
|
+
const members = await takeRankMembers(db, tableName, indexName, batchSize);
|
|
1483
|
+
if (members.length > 0) {
|
|
1484
|
+
for (const member of members) {
|
|
1485
|
+
if (member.rankKey !== void 0) await aggregate.deleteIfExists(ctx, {
|
|
1486
|
+
id: member.docId,
|
|
1487
|
+
key: member.rankKey,
|
|
1488
|
+
namespace: member.rankNamespace
|
|
1489
|
+
});
|
|
1490
|
+
await db.delete(AGGREGATE_MEMBER_TABLE, member._id);
|
|
1491
|
+
}
|
|
1492
|
+
return {
|
|
1493
|
+
done: false,
|
|
1494
|
+
processed: members.length
|
|
1495
|
+
};
|
|
1496
|
+
}
|
|
1497
|
+
const done = await aggregate.deleteTrees(ctx, RANK_TREE_DROP_BATCH);
|
|
1498
|
+
return {
|
|
1499
|
+
done,
|
|
1500
|
+
processed: done ? 0 : RANK_TREE_DROP_BATCH
|
|
1501
|
+
};
|
|
1502
|
+
};
|
|
1503
|
+
const reconcileRankMembership = async (db, params) => {
|
|
1504
|
+
const { tableName, definition, docId, doc } = params;
|
|
1505
|
+
const existing = await getRankMemberByDoc(db, tableName, definition.name, docId);
|
|
1506
|
+
const aggregate = rankAggregate(tableName, definition.name);
|
|
1507
|
+
const ctx = rankCtx(db);
|
|
1508
|
+
const now = Date.now();
|
|
1509
|
+
if (!doc) {
|
|
1510
|
+
if (existing?.rankKey !== void 0) {
|
|
1511
|
+
await aggregate.delete(ctx, {
|
|
1512
|
+
id: docId,
|
|
1513
|
+
key: existing.rankKey,
|
|
1514
|
+
namespace: existing.rankNamespace
|
|
1515
|
+
});
|
|
1516
|
+
await db.delete(AGGREGATE_MEMBER_TABLE, existing._id);
|
|
1517
|
+
}
|
|
1518
|
+
return;
|
|
1519
|
+
}
|
|
1520
|
+
const namespace = buildNamespace(definition, new Map(definition.partitionFields.map((field) => [field, doc[field]])));
|
|
1521
|
+
const key = getRankSortKey(definition, doc, docId);
|
|
1522
|
+
const sumValue = getRankSumValue(definition, doc);
|
|
1523
|
+
if (existing && stableEquals(existing.rankNamespace, namespace) && stableEquals(existing.rankKey, key) && (existing.rankSumValue ?? 0) === sumValue) return;
|
|
1524
|
+
if (existing?.rankKey !== void 0) {
|
|
1525
|
+
await aggregate.replace(ctx, {
|
|
1526
|
+
id: docId,
|
|
1527
|
+
key: existing.rankKey,
|
|
1528
|
+
namespace: existing.rankNamespace
|
|
1529
|
+
}, {
|
|
1530
|
+
key,
|
|
1531
|
+
namespace,
|
|
1532
|
+
sumValue
|
|
1533
|
+
});
|
|
1534
|
+
await db.patch(AGGREGATE_MEMBER_TABLE, existing._id, {
|
|
1535
|
+
kind: RANK_MEMBER_KIND,
|
|
1536
|
+
keyHash: "__rank__",
|
|
1537
|
+
keyParts: [],
|
|
1538
|
+
sumValues: {},
|
|
1539
|
+
nonNullCountValues: {},
|
|
1540
|
+
extremaValues: {},
|
|
1541
|
+
rankNamespace: namespace,
|
|
1542
|
+
rankKey: key,
|
|
1543
|
+
rankSumValue: sumValue,
|
|
1544
|
+
updatedAt: now
|
|
1545
|
+
});
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1548
|
+
await aggregate.insert(ctx, {
|
|
1549
|
+
id: docId,
|
|
1550
|
+
key,
|
|
1551
|
+
namespace,
|
|
1552
|
+
sumValue
|
|
1553
|
+
});
|
|
1554
|
+
await db.insert(AGGREGATE_MEMBER_TABLE, {
|
|
1555
|
+
kind: RANK_MEMBER_KIND,
|
|
1556
|
+
tableKey: tableName,
|
|
1557
|
+
indexName: definition.name,
|
|
1558
|
+
docId,
|
|
1559
|
+
keyHash: "__rank__",
|
|
1560
|
+
keyParts: [],
|
|
1561
|
+
sumValues: {},
|
|
1562
|
+
nonNullCountValues: {},
|
|
1563
|
+
extremaValues: {},
|
|
1564
|
+
rankNamespace: namespace,
|
|
1565
|
+
rankKey: key,
|
|
1566
|
+
rankSumValue: sumValue,
|
|
1567
|
+
updatedAt: now
|
|
1568
|
+
});
|
|
1569
|
+
};
|
|
1570
|
+
const applyRankIndexesForChange = async (db, tableName, rankIndexes, change) => {
|
|
1571
|
+
const docId = String(change.id);
|
|
1572
|
+
for (const rankIndex of rankIndexes) await reconcileRankMembership(db, {
|
|
1573
|
+
tableName,
|
|
1574
|
+
definition: rankIndex,
|
|
1575
|
+
docId,
|
|
1576
|
+
doc: change.operation === "delete" ? null : change.newDoc
|
|
1577
|
+
});
|
|
1578
|
+
};
|
|
1579
|
+
const readRankCount = async (db, plan) => {
|
|
1580
|
+
return await rankAggregate(plan.tableName, plan.indexName).count(rankCtx(db), { namespace: plan.namespace });
|
|
1581
|
+
};
|
|
1582
|
+
const readRankSum = async (db, plan) => {
|
|
1583
|
+
return await rankAggregate(plan.tableName, plan.indexName).sum(rankCtx(db), { namespace: plan.namespace });
|
|
1584
|
+
};
|
|
1585
|
+
const toPublicRankItem = (plan, item) => ({
|
|
1586
|
+
id: item.id,
|
|
1587
|
+
key: toPublicRankKey(plan.definition, item.key),
|
|
1588
|
+
sumValue: item.sumValue
|
|
1589
|
+
});
|
|
1590
|
+
const readRankAtWithCount = async (db, plan, offset, count) => {
|
|
1591
|
+
if (count <= 0) return null;
|
|
1592
|
+
const normalizedOffset = offset < 0 ? count + offset : offset;
|
|
1593
|
+
if (normalizedOffset < 0 || normalizedOffset >= count) return null;
|
|
1594
|
+
return toPublicRankItem(plan, await rankAggregate(plan.tableName, plan.indexName).at(rankCtx(db), offset, { namespace: plan.namespace }));
|
|
1595
|
+
};
|
|
1596
|
+
const readRankAt = async (db, plan, offset) => await readRankAtWithCount(db, plan, offset, await readRankCount(db, plan));
|
|
1597
|
+
const readRankIndexOf = async (db, plan, args) => {
|
|
1598
|
+
const member = await getRankMemberByDoc(db, plan.tableName, plan.indexName, args.id);
|
|
1599
|
+
if (!member || member.rankKey === void 0) return -1;
|
|
1600
|
+
if (!stableEquals(member.rankNamespace, plan.namespace)) return -1;
|
|
1601
|
+
return await rankAggregate(plan.tableName, plan.indexName).indexOf(rankCtx(db), member.rankKey, {
|
|
1602
|
+
namespace: plan.namespace,
|
|
1603
|
+
id: args.id
|
|
1604
|
+
});
|
|
1605
|
+
};
|
|
1606
|
+
const readRankPaginate = async (db, plan, cursor, limit) => {
|
|
1607
|
+
const paged = await rankAggregate(plan.tableName, plan.indexName).paginate(rankCtx(db), {
|
|
1608
|
+
namespace: plan.namespace,
|
|
1609
|
+
cursor: cursor ?? void 0,
|
|
1610
|
+
pageSize: limit
|
|
1611
|
+
});
|
|
1612
|
+
return {
|
|
1613
|
+
continueCursor: paged.cursor ?? "",
|
|
1614
|
+
isDone: paged.isDone,
|
|
1615
|
+
page: paged.page.map((item) => toPublicRankItem(plan, item))
|
|
1616
|
+
};
|
|
1617
|
+
};
|
|
1618
|
+
const readRankMin = async (db, plan) => {
|
|
1619
|
+
const item = await rankAggregate(plan.tableName, plan.indexName).min(rankCtx(db), { namespace: plan.namespace });
|
|
1620
|
+
return item ? toPublicRankItem(plan, item) : null;
|
|
1621
|
+
};
|
|
1622
|
+
const readRankMax = async (db, plan) => {
|
|
1623
|
+
const item = await rankAggregate(plan.tableName, plan.indexName).max(rankCtx(db), { namespace: plan.namespace });
|
|
1624
|
+
return item ? toPublicRankItem(plan, item) : null;
|
|
1625
|
+
};
|
|
1626
|
+
const readRankRandom = async (db, plan) => {
|
|
1627
|
+
const count = await readRankCount(db, plan);
|
|
1628
|
+
if (count <= 0) return null;
|
|
1629
|
+
return await readRankAtWithCount(db, plan, Math.floor(Math.random() * count), count);
|
|
1630
|
+
};
|
|
1631
|
+
|
|
1632
|
+
//#endregion
|
|
1633
|
+
//#region src/orm/aggregate-index/backfill.ts
|
|
1634
|
+
const DEFAULT_BACKFILL_BATCH_SIZE = 1e3;
|
|
1635
|
+
const getBackfillMode = (candidate) => {
|
|
1636
|
+
if (candidate === void 0) return "resume";
|
|
1637
|
+
if (candidate === "resume" || candidate === "rebuild" || candidate === "prune") return candidate;
|
|
1638
|
+
throw new Error("countBackfill mode must be one of 'resume', 'rebuild', or 'prune'.");
|
|
1639
|
+
};
|
|
1640
|
+
const getBackfillBatchSize = (candidate) => {
|
|
1641
|
+
if (candidate === void 0) return DEFAULT_BACKFILL_BATCH_SIZE;
|
|
1642
|
+
if (typeof candidate !== "number" || !Number.isInteger(candidate)) throw new Error("countBackfill batchSize must be a positive integer.");
|
|
1643
|
+
if (candidate < 1) throw new Error("countBackfill batchSize must be a positive integer.");
|
|
1644
|
+
return candidate;
|
|
1645
|
+
};
|
|
1646
|
+
const matchesFilter = (target, args) => {
|
|
1647
|
+
if (args.tableName && target.tableName !== args.tableName) return false;
|
|
1648
|
+
if (args.indexName && target.indexName !== args.indexName) return false;
|
|
1649
|
+
return true;
|
|
1650
|
+
};
|
|
1651
|
+
const getTargets = (schema, args) => {
|
|
1652
|
+
const metricTargets = listSchemaAggregateIndexes(schema).map((entry) => ({
|
|
1653
|
+
kind: "metric",
|
|
1654
|
+
tableName: entry.tableName,
|
|
1655
|
+
indexName: entry.indexName,
|
|
1656
|
+
fields: entry.fields,
|
|
1657
|
+
countFields: entry.countFields,
|
|
1658
|
+
sumFields: entry.sumFields,
|
|
1659
|
+
avgFields: entry.avgFields,
|
|
1660
|
+
minFields: entry.minFields,
|
|
1661
|
+
maxFields: entry.maxFields
|
|
1662
|
+
}));
|
|
1663
|
+
const rankTargets = [];
|
|
1664
|
+
for (const tableConfig of Object.values(schema)) {
|
|
1665
|
+
const rankIndexes = getRankIndexDefinitions(tableConfig);
|
|
1666
|
+
for (const rankIndex of rankIndexes) rankTargets.push({
|
|
1667
|
+
kind: "rank",
|
|
1668
|
+
tableName: tableConfig.name,
|
|
1669
|
+
indexName: rankIndex.name,
|
|
1670
|
+
fields: rankIndex.partitionFields,
|
|
1671
|
+
orderFields: rankIndex.orderFields,
|
|
1672
|
+
rankSumField: rankIndex.sumField
|
|
1673
|
+
});
|
|
1674
|
+
}
|
|
1675
|
+
return [...metricTargets, ...rankTargets].filter((entry) => matchesFilter(entry, args));
|
|
1676
|
+
};
|
|
1677
|
+
const dedupe = (fields) => [...new Set(fields)];
|
|
1678
|
+
const computeMetricDefinition = (target) => ({
|
|
1679
|
+
countFields: dedupe(target.countFields ?? []),
|
|
1680
|
+
sumFields: dedupe(target.sumFields ?? []),
|
|
1681
|
+
avgFields: dedupe(target.avgFields ?? []),
|
|
1682
|
+
minFields: dedupe(target.minFields ?? []),
|
|
1683
|
+
maxFields: dedupe(target.maxFields ?? [])
|
|
1684
|
+
});
|
|
1685
|
+
const computeKeyDefinitionHash = (target) => JSON.stringify({
|
|
1686
|
+
kind: target.kind,
|
|
1687
|
+
fields: target.fields,
|
|
1688
|
+
orderFields: target.orderFields ?? []
|
|
1689
|
+
});
|
|
1690
|
+
const computeMetricDefinitionHash = (target) => target.kind === "rank" ? JSON.stringify({ sumField: target.rankSumField ?? null }) : JSON.stringify(computeMetricDefinition(target));
|
|
1691
|
+
const parseMetricDefinitionHash = (metricDefinitionHash) => {
|
|
1692
|
+
let parsed;
|
|
1693
|
+
try {
|
|
1694
|
+
parsed = JSON.parse(metricDefinitionHash);
|
|
1695
|
+
} catch {
|
|
1696
|
+
throw new Error(`Invalid metricDefinitionHash payload: ${metricDefinitionHash}`);
|
|
1697
|
+
}
|
|
1698
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("Invalid metricDefinitionHash payload shape.");
|
|
1699
|
+
const pick = (key) => {
|
|
1700
|
+
const candidate = parsed[key];
|
|
1701
|
+
if (!Array.isArray(candidate)) throw new Error(`metricDefinitionHash is missing '${key}'.`);
|
|
1702
|
+
if (candidate.some((entry) => typeof entry !== "string")) throw new Error(`metricDefinitionHash has non-string value in '${key}'.`);
|
|
1703
|
+
return dedupe(candidate);
|
|
1704
|
+
};
|
|
1705
|
+
return {
|
|
1706
|
+
countFields: pick("countFields"),
|
|
1707
|
+
sumFields: pick("sumFields"),
|
|
1708
|
+
avgFields: pick("avgFields"),
|
|
1709
|
+
minFields: pick("minFields"),
|
|
1710
|
+
maxFields: pick("maxFields")
|
|
1711
|
+
};
|
|
1712
|
+
};
|
|
1713
|
+
const toMetricStorageDefinition = (metricDefinition) => ({
|
|
1714
|
+
sumFields: new Set([...metricDefinition.sumFields, ...metricDefinition.avgFields]),
|
|
1715
|
+
nonNullCountFields: new Set([...metricDefinition.countFields, ...metricDefinition.avgFields]),
|
|
1716
|
+
extremaFields: new Set([...metricDefinition.minFields, ...metricDefinition.maxFields])
|
|
1717
|
+
});
|
|
1718
|
+
const requiresMetricBackfill = (existingMetricDefinitionHash, targetMetricDefinitionHash) => {
|
|
1719
|
+
const existingDefinition = parseMetricDefinitionHash(existingMetricDefinitionHash);
|
|
1720
|
+
const targetDefinition = parseMetricDefinitionHash(targetMetricDefinitionHash);
|
|
1721
|
+
const existingStorage = toMetricStorageDefinition(existingDefinition);
|
|
1722
|
+
const targetStorage = toMetricStorageDefinition(targetDefinition);
|
|
1723
|
+
const hasMissing = (target, existing) => {
|
|
1724
|
+
for (const field of target) if (!existing.has(field)) return true;
|
|
1725
|
+
return false;
|
|
1726
|
+
};
|
|
1727
|
+
return hasMissing(targetStorage.sumFields, existingStorage.sumFields) || hasMissing(targetStorage.nonNullCountFields, existingStorage.nonNullCountFields) || hasMissing(targetStorage.extremaFields, existingStorage.extremaFields);
|
|
1728
|
+
};
|
|
1729
|
+
function createCountBackfillHandlers(schema, getChunkRef) {
|
|
1730
|
+
const serializeKey = (kind, tableName, indexName) => `${kind}\u0000${tableName}\u0000${indexName}`;
|
|
1731
|
+
const stateKindFor = (kind) => kind === "rank" ? AGGREGATE_STATE_KIND_RANK : AGGREGATE_STATE_KIND_METRIC;
|
|
1732
|
+
const scheduleChunk = async (ctx, tableName, indexName, batchSize) => {
|
|
1733
|
+
const chunkRef = getChunkRef?.();
|
|
1734
|
+
if (ctx.scheduler && chunkRef) await ctx.scheduler.runAfter(0, chunkRef, {
|
|
1735
|
+
tableName,
|
|
1736
|
+
indexName,
|
|
1737
|
+
batchSize
|
|
1738
|
+
});
|
|
1739
|
+
};
|
|
1740
|
+
/**
|
|
1741
|
+
* Drains an index's stored state until it is empty or the shared per-mutation
|
|
1742
|
+
* budget runs out. Returns true only when the index is fully cleared, so the
|
|
1743
|
+
* caller knows whether to advance the state machine or reschedule.
|
|
1744
|
+
*/
|
|
1745
|
+
const drainIndexClear = async (ctx, kind, tableName, indexName, batchSize, budget) => {
|
|
1746
|
+
while (budget.remaining > 0) {
|
|
1747
|
+
const limit = Math.min(batchSize, budget.remaining);
|
|
1748
|
+
const step = kind === "rank" ? await clearRankIndexChunk(ctx.db, tableName, indexName, limit) : await clearCountIndexChunk(ctx.db, tableName, indexName, limit);
|
|
1749
|
+
if (step.done) return true;
|
|
1750
|
+
budget.remaining -= Math.max(step.processed, 1);
|
|
1751
|
+
}
|
|
1752
|
+
return false;
|
|
1753
|
+
};
|
|
1754
|
+
const pruneRemovedState = async (ctx, args, targets, batchSize, clearBudget) => {
|
|
1755
|
+
const targetKeys = new Set(targets.map((target) => serializeKey(target.kind, target.tableName, target.indexName)));
|
|
1756
|
+
const states = await ctx.db.query(AGGREGATE_STATE_TABLE).collect();
|
|
1757
|
+
const stateByKey = new Map(states.map((state) => [serializeKey(state.kind === AGGREGATE_STATE_KIND_RANK ? "rank" : "metric", state.tableKey, state.indexName), state]));
|
|
1758
|
+
const existingKeys = /* @__PURE__ */ new Set();
|
|
1759
|
+
for (const state of states) existingKeys.add(serializeKey(state.kind === AGGREGATE_STATE_KIND_RANK ? "rank" : "metric", state.tableKey, state.indexName));
|
|
1760
|
+
if (args.tableName && args.indexName) {
|
|
1761
|
+
const tableName = args.tableName;
|
|
1762
|
+
const indexName = args.indexName;
|
|
1763
|
+
const [bucket, extrema, metricMember, rankMember] = await Promise.all([
|
|
1764
|
+
ctx.db.query(AGGREGATE_BUCKET_TABLE).withIndex("by_table_index", (q) => q.eq("tableKey", tableName).eq("indexName", indexName)).first(),
|
|
1765
|
+
ctx.db.query(AGGREGATE_EXTREMA_TABLE).withIndex("by_table_index", (q) => q.eq("tableKey", tableName).eq("indexName", indexName)).first(),
|
|
1766
|
+
ctx.db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index", (q) => q.eq("kind", AGGREGATE_STATE_KIND_METRIC).eq("tableKey", tableName).eq("indexName", indexName)).first(),
|
|
1767
|
+
ctx.db.query(AGGREGATE_MEMBER_TABLE).withIndex("by_kind_table_index", (q) => q.eq("kind", AGGREGATE_STATE_KIND_RANK).eq("tableKey", tableName).eq("indexName", indexName)).first()
|
|
1768
|
+
]);
|
|
1769
|
+
if (bucket || extrema || metricMember) existingKeys.add(serializeKey("metric", tableName, indexName));
|
|
1770
|
+
if (rankMember) existingKeys.add(serializeKey("rank", tableName, indexName));
|
|
1771
|
+
}
|
|
1772
|
+
let pruned = 0;
|
|
1773
|
+
let pruning = 0;
|
|
1774
|
+
for (const key of existingKeys) {
|
|
1775
|
+
const [kind, tableName, indexName] = key.split("\0");
|
|
1776
|
+
if (args.tableName && tableName !== args.tableName) continue;
|
|
1777
|
+
if (args.indexName && indexName !== args.indexName) continue;
|
|
1778
|
+
if (targetKeys.has(key)) continue;
|
|
1779
|
+
const state = stateByKey.get(key);
|
|
1780
|
+
if (!await drainIndexClear(ctx, kind, tableName, indexName, batchSize, clearBudget)) {
|
|
1781
|
+
const now = Date.now();
|
|
1782
|
+
await setCountState(ctx.db, {
|
|
1783
|
+
tableName,
|
|
1784
|
+
indexName,
|
|
1785
|
+
kind: stateKindFor(kind),
|
|
1786
|
+
keyDefinitionHash: state?.keyDefinitionHash ?? "",
|
|
1787
|
+
metricDefinitionHash: state?.metricDefinitionHash ?? "",
|
|
1788
|
+
status: COUNT_STATUS_CLEARING,
|
|
1789
|
+
cursor: null,
|
|
1790
|
+
processed: 0,
|
|
1791
|
+
startedAt: now,
|
|
1792
|
+
updatedAt: now,
|
|
1793
|
+
completedAt: null,
|
|
1794
|
+
lastError: null
|
|
1795
|
+
}, stateKindFor(kind));
|
|
1796
|
+
await scheduleChunk(ctx, tableName, indexName, batchSize);
|
|
1797
|
+
pruning += 1;
|
|
1798
|
+
continue;
|
|
1799
|
+
}
|
|
1800
|
+
if (state) await ctx.db.delete(AGGREGATE_STATE_TABLE, state._id);
|
|
1801
|
+
pruned += 1;
|
|
1802
|
+
}
|
|
1803
|
+
return {
|
|
1804
|
+
pruned,
|
|
1805
|
+
pruning
|
|
1806
|
+
};
|
|
1807
|
+
};
|
|
1808
|
+
/**
|
|
1809
|
+
* Finishes clearing an index that no longer exists in the schema. Orphans have
|
|
1810
|
+
* no backfill target, so they are driven purely by their CLEARING state row.
|
|
1811
|
+
* Returns true when this invocation handled one.
|
|
1812
|
+
*/
|
|
1813
|
+
const drainOrphanClear = async (ctx, args, targets, batchSize) => {
|
|
1814
|
+
const targetKeys = new Set(targets.map((target) => serializeKey(target.kind, target.tableName, target.indexName)));
|
|
1815
|
+
const states = await ctx.db.query(AGGREGATE_STATE_TABLE).collect();
|
|
1816
|
+
for (const state of states) {
|
|
1817
|
+
if (state.status !== COUNT_STATUS_CLEARING) continue;
|
|
1818
|
+
const kind = state.kind === AGGREGATE_STATE_KIND_RANK ? "rank" : "metric";
|
|
1819
|
+
if (targetKeys.has(serializeKey(kind, state.tableKey, state.indexName))) continue;
|
|
1820
|
+
if (args.tableName && state.tableKey !== args.tableName) continue;
|
|
1821
|
+
if (args.indexName && state.indexName !== args.indexName) continue;
|
|
1822
|
+
if (await drainIndexClear(ctx, kind, state.tableKey, state.indexName, batchSize, { remaining: batchSize })) {
|
|
1823
|
+
await ctx.db.delete(AGGREGATE_STATE_TABLE, state._id);
|
|
1824
|
+
return true;
|
|
1825
|
+
}
|
|
1826
|
+
await ctx.db.patch(AGGREGATE_STATE_TABLE, state._id, { updatedAt: Date.now() });
|
|
1827
|
+
await scheduleChunk(ctx, state.tableKey, state.indexName, batchSize);
|
|
1828
|
+
return true;
|
|
1829
|
+
}
|
|
1830
|
+
return false;
|
|
1831
|
+
};
|
|
1832
|
+
const kickoff = async (ctx, args) => {
|
|
1833
|
+
const targets = getTargets(schema, args);
|
|
1834
|
+
const mode = getBackfillMode(args.mode);
|
|
1835
|
+
const batchSize = getBackfillBatchSize(args.batchSize);
|
|
1836
|
+
const clearBudget = { remaining: batchSize };
|
|
1837
|
+
const { pruned, pruning } = await pruneRemovedState(ctx, args, targets, batchSize, clearBudget);
|
|
1838
|
+
if (mode === "prune") return {
|
|
1839
|
+
targets: targets.length,
|
|
1840
|
+
mode,
|
|
1841
|
+
scheduled: 0,
|
|
1842
|
+
skippedReady: 0,
|
|
1843
|
+
needsRebuild: 0,
|
|
1844
|
+
pruned,
|
|
1845
|
+
pruning,
|
|
1846
|
+
status: "ok"
|
|
1847
|
+
};
|
|
1848
|
+
const now = Date.now();
|
|
1849
|
+
let scheduled = 0;
|
|
1850
|
+
let skippedReady = 0;
|
|
1851
|
+
let needsRebuild = 0;
|
|
1852
|
+
for (const target of targets) {
|
|
1853
|
+
const stateKind = target.kind === "rank" ? AGGREGATE_STATE_KIND_RANK : AGGREGATE_STATE_KIND_METRIC;
|
|
1854
|
+
const existing = await getCountState(ctx.db, target.tableName, target.indexName, stateKind);
|
|
1855
|
+
const keyDefinitionHash = computeKeyDefinitionHash(target);
|
|
1856
|
+
const metricDefinitionHash = computeMetricDefinitionHash(target);
|
|
1857
|
+
const nextStateBase = {
|
|
1858
|
+
tableName: target.tableName,
|
|
1859
|
+
indexName: target.indexName,
|
|
1860
|
+
keyDefinitionHash,
|
|
1861
|
+
metricDefinitionHash
|
|
1862
|
+
};
|
|
1863
|
+
if (mode === "rebuild") {
|
|
1864
|
+
const cleared = await drainIndexClear(ctx, target.kind, target.tableName, target.indexName, batchSize, clearBudget);
|
|
1865
|
+
await setCountState(ctx.db, {
|
|
1866
|
+
...nextStateBase,
|
|
1867
|
+
kind: stateKind,
|
|
1868
|
+
status: cleared ? COUNT_STATUS_BUILDING : COUNT_STATUS_CLEARING,
|
|
1869
|
+
cursor: null,
|
|
1870
|
+
processed: 0,
|
|
1871
|
+
startedAt: now,
|
|
1872
|
+
updatedAt: now,
|
|
1873
|
+
completedAt: null,
|
|
1874
|
+
lastError: null
|
|
1875
|
+
}, stateKind);
|
|
1876
|
+
} else if (existing) {
|
|
1877
|
+
if (existing.keyDefinitionHash !== keyDefinitionHash) {
|
|
1878
|
+
needsRebuild += 1;
|
|
1879
|
+
continue;
|
|
1880
|
+
}
|
|
1881
|
+
if (existing.metricDefinitionHash !== metricDefinitionHash) {
|
|
1882
|
+
if (!(target.kind === "rank" ? true : requiresMetricBackfill(existing.metricDefinitionHash, metricDefinitionHash)) && existing.status === COUNT_STATUS_READY) {
|
|
1883
|
+
await setCountState(ctx.db, {
|
|
1884
|
+
...nextStateBase,
|
|
1885
|
+
kind: stateKind,
|
|
1886
|
+
status: COUNT_STATUS_READY,
|
|
1887
|
+
cursor: null,
|
|
1888
|
+
processed: existing.processed,
|
|
1889
|
+
startedAt: existing.startedAt,
|
|
1890
|
+
updatedAt: now,
|
|
1891
|
+
completedAt: existing.completedAt ?? now,
|
|
1892
|
+
lastError: null
|
|
1893
|
+
}, stateKind);
|
|
1894
|
+
skippedReady += 1;
|
|
1895
|
+
continue;
|
|
1896
|
+
}
|
|
1897
|
+
await setCountState(ctx.db, {
|
|
1898
|
+
...nextStateBase,
|
|
1899
|
+
kind: stateKind,
|
|
1900
|
+
status: COUNT_STATUS_BUILDING,
|
|
1901
|
+
cursor: null,
|
|
1902
|
+
processed: 0,
|
|
1903
|
+
startedAt: now,
|
|
1904
|
+
updatedAt: now,
|
|
1905
|
+
completedAt: null,
|
|
1906
|
+
lastError: null
|
|
1907
|
+
}, stateKind);
|
|
1908
|
+
} else if (existing.status === COUNT_STATUS_READY) {
|
|
1909
|
+
skippedReady += 1;
|
|
1910
|
+
continue;
|
|
1911
|
+
}
|
|
1912
|
+
} else await setCountState(ctx.db, {
|
|
1913
|
+
...nextStateBase,
|
|
1914
|
+
kind: stateKind,
|
|
1915
|
+
status: COUNT_STATUS_BUILDING,
|
|
1916
|
+
cursor: null,
|
|
1917
|
+
processed: 0,
|
|
1918
|
+
startedAt: now,
|
|
1919
|
+
updatedAt: now,
|
|
1920
|
+
completedAt: null,
|
|
1921
|
+
lastError: null
|
|
1922
|
+
}, stateKind);
|
|
1923
|
+
await scheduleChunk(ctx, target.tableName, target.indexName, batchSize);
|
|
1924
|
+
scheduled += 1;
|
|
1925
|
+
}
|
|
1926
|
+
return {
|
|
1927
|
+
targets: targets.length,
|
|
1928
|
+
mode,
|
|
1929
|
+
scheduled,
|
|
1930
|
+
skippedReady,
|
|
1931
|
+
needsRebuild,
|
|
1932
|
+
pruned,
|
|
1933
|
+
pruning,
|
|
1934
|
+
status: "ok"
|
|
1935
|
+
};
|
|
1936
|
+
};
|
|
1937
|
+
const chunk = async (ctx, args) => {
|
|
1938
|
+
const batchSize = getBackfillBatchSize(args.batchSize);
|
|
1939
|
+
const targets = getTargets(schema, args);
|
|
1940
|
+
if (!(Boolean(args.tableName && args.indexName) && targets.length === 1) && await drainOrphanClear(ctx, args, targets, batchSize)) return { status: "ok" };
|
|
1941
|
+
if (targets.length > 1) {
|
|
1942
|
+
for (const target of targets) {
|
|
1943
|
+
const stateKind = target.kind === "rank" ? AGGREGATE_STATE_KIND_RANK : AGGREGATE_STATE_KIND_METRIC;
|
|
1944
|
+
const state = await getCountState(ctx.db, target.tableName, target.indexName, stateKind);
|
|
1945
|
+
if (!state || state.status !== COUNT_STATUS_READY) return chunk(ctx, {
|
|
1946
|
+
tableName: target.tableName,
|
|
1947
|
+
indexName: target.indexName,
|
|
1948
|
+
batchSize
|
|
1949
|
+
});
|
|
1950
|
+
}
|
|
1951
|
+
return { status: "ok" };
|
|
1952
|
+
}
|
|
1953
|
+
for (const target of targets) {
|
|
1954
|
+
const stateKind = target.kind === "rank" ? AGGREGATE_STATE_KIND_RANK : AGGREGATE_STATE_KIND_METRIC;
|
|
1955
|
+
try {
|
|
1956
|
+
const state = await getCountState(ctx.db, target.tableName, target.indexName, stateKind);
|
|
1957
|
+
if (!state || state.status === COUNT_STATUS_READY) continue;
|
|
1958
|
+
if (state.status === COUNT_STATUS_CLEARING) {
|
|
1959
|
+
const cleared = await drainIndexClear(ctx, target.kind, target.tableName, target.indexName, batchSize, { remaining: batchSize });
|
|
1960
|
+
const clearedAt = Date.now();
|
|
1961
|
+
await setCountState(ctx.db, {
|
|
1962
|
+
tableName: target.tableName,
|
|
1963
|
+
indexName: target.indexName,
|
|
1964
|
+
kind: stateKind,
|
|
1965
|
+
keyDefinitionHash: state.keyDefinitionHash,
|
|
1966
|
+
metricDefinitionHash: state.metricDefinitionHash,
|
|
1967
|
+
status: cleared ? COUNT_STATUS_BUILDING : COUNT_STATUS_CLEARING,
|
|
1968
|
+
cursor: null,
|
|
1969
|
+
processed: 0,
|
|
1970
|
+
startedAt: state.startedAt,
|
|
1971
|
+
updatedAt: clearedAt,
|
|
1972
|
+
completedAt: null,
|
|
1973
|
+
lastError: null
|
|
1974
|
+
}, stateKind);
|
|
1975
|
+
await scheduleChunk(ctx, target.tableName, target.indexName, batchSize);
|
|
1976
|
+
continue;
|
|
1977
|
+
}
|
|
1978
|
+
const cursor = state.cursor ?? null;
|
|
1979
|
+
const page = await ctx.db.query(target.tableName).withIndex("by_creation_time").paginate({
|
|
1980
|
+
cursor,
|
|
1981
|
+
numItems: batchSize
|
|
1982
|
+
});
|
|
1983
|
+
const pendingDeltas = [];
|
|
1984
|
+
for (const doc of page.page) if (target.kind === "rank") await reconcileRankMembership(ctx.db, {
|
|
1985
|
+
tableName: target.tableName,
|
|
1986
|
+
definition: {
|
|
1987
|
+
name: target.indexName,
|
|
1988
|
+
partitionFields: target.fields,
|
|
1989
|
+
orderFields: target.orderFields ?? [],
|
|
1990
|
+
sumField: target.rankSumField
|
|
1991
|
+
},
|
|
1992
|
+
docId: String(doc._id),
|
|
1993
|
+
doc
|
|
1994
|
+
});
|
|
1995
|
+
else pendingDeltas.push(await computeAggregateMembershipDelta(ctx.db, {
|
|
1996
|
+
tableName: target.tableName,
|
|
1997
|
+
indexName: target.indexName,
|
|
1998
|
+
docId: String(doc._id),
|
|
1999
|
+
keyParts: computeCountKeyParts(doc, target.fields),
|
|
2000
|
+
metricValues: computeAggregateMetricValues(doc, {
|
|
2001
|
+
name: target.indexName,
|
|
2002
|
+
fields: target.fields,
|
|
2003
|
+
countFields: target.countFields ?? [],
|
|
2004
|
+
sumFields: target.sumFields ?? [],
|
|
2005
|
+
avgFields: target.avgFields ?? [],
|
|
2006
|
+
minFields: target.minFields ?? [],
|
|
2007
|
+
maxFields: target.maxFields ?? []
|
|
2008
|
+
})
|
|
2009
|
+
}));
|
|
2010
|
+
if (pendingDeltas.length > 0) await flushAggregateMembershipDeltas(ctx.db, target.tableName, target.indexName, pendingDeltas);
|
|
2011
|
+
const now = Date.now();
|
|
2012
|
+
const nextProcessed = state.processed + page.page.length;
|
|
2013
|
+
if (page.isDone) {
|
|
2014
|
+
await setCountState(ctx.db, {
|
|
2015
|
+
tableName: target.tableName,
|
|
2016
|
+
indexName: target.indexName,
|
|
2017
|
+
kind: stateKind,
|
|
2018
|
+
keyDefinitionHash: state.keyDefinitionHash,
|
|
2019
|
+
metricDefinitionHash: state.metricDefinitionHash,
|
|
2020
|
+
status: COUNT_STATUS_READY,
|
|
2021
|
+
cursor: null,
|
|
2022
|
+
processed: nextProcessed,
|
|
2023
|
+
startedAt: state.startedAt,
|
|
2024
|
+
updatedAt: now,
|
|
2025
|
+
completedAt: now,
|
|
2026
|
+
lastError: null
|
|
2027
|
+
}, stateKind);
|
|
2028
|
+
continue;
|
|
2029
|
+
}
|
|
2030
|
+
await setCountState(ctx.db, {
|
|
2031
|
+
tableName: target.tableName,
|
|
2032
|
+
indexName: target.indexName,
|
|
2033
|
+
kind: stateKind,
|
|
2034
|
+
keyDefinitionHash: state.keyDefinitionHash,
|
|
2035
|
+
metricDefinitionHash: state.metricDefinitionHash,
|
|
2036
|
+
status: COUNT_STATUS_BUILDING,
|
|
2037
|
+
cursor: page.continueCursor,
|
|
2038
|
+
processed: nextProcessed,
|
|
2039
|
+
startedAt: state.startedAt,
|
|
2040
|
+
updatedAt: now,
|
|
2041
|
+
completedAt: null,
|
|
2042
|
+
lastError: null
|
|
2043
|
+
}, stateKind);
|
|
2044
|
+
await scheduleChunk(ctx, target.tableName, target.indexName, batchSize);
|
|
2045
|
+
} catch (error) {
|
|
2046
|
+
await setCountStateError(ctx.db, target.tableName, target.indexName, error, stateKind);
|
|
2047
|
+
throw error;
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
return { status: "ok" };
|
|
2051
|
+
};
|
|
2052
|
+
const status = async (ctx, args) => {
|
|
2053
|
+
const states = await ctx.db.query(AGGREGATE_STATE_TABLE).collect();
|
|
2054
|
+
const statesByKey = new Map(states.map((entry) => [`${entry.kind === AGGREGATE_STATE_KIND_RANK ? "rank" : "metric"}:${entry.tableKey}:${entry.indexName}`, entry]));
|
|
2055
|
+
return getTargets(schema, args).map((target) => {
|
|
2056
|
+
const key = `${target.kind}:${target.tableName}:${target.indexName}`;
|
|
2057
|
+
const entry = statesByKey.get(key);
|
|
2058
|
+
if (!entry) return {
|
|
2059
|
+
kind: target.kind === "rank" ? AGGREGATE_STATE_KIND_RANK : AGGREGATE_STATE_KIND_METRIC,
|
|
2060
|
+
tableName: target.tableName,
|
|
2061
|
+
indexName: target.indexName,
|
|
2062
|
+
keyDefinitionHash: computeKeyDefinitionHash(target),
|
|
2063
|
+
metricDefinitionHash: computeMetricDefinitionHash(target),
|
|
2064
|
+
status: COUNT_STATUS_BUILDING,
|
|
2065
|
+
cursor: null,
|
|
2066
|
+
processed: 0,
|
|
2067
|
+
startedAt: 0,
|
|
2068
|
+
updatedAt: 0,
|
|
2069
|
+
completedAt: null,
|
|
2070
|
+
lastError: null
|
|
2071
|
+
};
|
|
2072
|
+
const { tableKey, ...rest } = entry;
|
|
2073
|
+
return {
|
|
2074
|
+
...rest,
|
|
2075
|
+
tableName: tableKey
|
|
2076
|
+
};
|
|
2077
|
+
});
|
|
2078
|
+
};
|
|
2079
|
+
return {
|
|
2080
|
+
kickoff,
|
|
2081
|
+
chunk,
|
|
2082
|
+
status
|
|
2083
|
+
};
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
//#endregion
|
|
2087
|
+
//#region src/orm/aggregate-index/capability.ts
|
|
2088
|
+
/**
|
|
2089
|
+
* Registers the aggregate-index runtime with `createOrm()`.
|
|
2090
|
+
*
|
|
2091
|
+
* Required by `aggregateIndex(...)` / `rankIndex(...)` schemas, by
|
|
2092
|
+
* `count()` / `aggregate()` / `rank()` queries, and by the backfill handlers
|
|
2093
|
+
* exposed through `orm.api()`. Importing this pulls the aggregate btree
|
|
2094
|
+
* runtime into the calling Convex module, so only register it where it is
|
|
2095
|
+
* actually used.
|
|
2096
|
+
*
|
|
2097
|
+
* Each entry forwards through the module namespace rather than capturing the
|
|
2098
|
+
* binding, so the resolved implementation is read per call. Snapshotting the
|
|
2099
|
+
* bindings here would freeze the capability at construction time and make the
|
|
2100
|
+
* runtime unobservable to anything that wraps a module export.
|
|
2101
|
+
*/
|
|
2102
|
+
const aggregateCapability = () => ({
|
|
2103
|
+
kind: "aggregate",
|
|
2104
|
+
value: {
|
|
2105
|
+
applyAggregateIndexesForChange: (...args) => applyAggregateIndexesForChange(...args),
|
|
2106
|
+
applyRankIndexesForChange: (...args) => applyRankIndexesForChange(...args),
|
|
2107
|
+
assertAggregateIndexesWritable: (...args) => assertAggregateIndexesWritable(...args),
|
|
2108
|
+
compileAggregateQueryPlan: (...args) => compileAggregateQueryPlan(...args),
|
|
2109
|
+
compileCountFieldQueryPlan: (...args) => compileCountFieldQueryPlan(...args),
|
|
2110
|
+
compileCountQueryPlan: (...args) => compileCountQueryPlan(...args),
|
|
2111
|
+
compileRankPlan: (...args) => compileRankPlan(...args),
|
|
2112
|
+
createCountBackfillHandlers: (...args) => createCountBackfillHandlers(...args),
|
|
2113
|
+
ensureAggregateIndexReady: (...args) => ensureAggregateIndexReady(...args),
|
|
2114
|
+
ensureCountIndexReady: (...args) => ensureCountIndexReady(...args),
|
|
2115
|
+
ensureRankAllowedForRls: (...args) => ensureRankAllowedForRls(...args),
|
|
2116
|
+
ensureRankIndexReady: (...args) => ensureRankIndexReady(...args),
|
|
2117
|
+
isAggregatePlanZero: (...args) => isAggregatePlanZero(...args),
|
|
2118
|
+
isIndexCountZero: (...args) => isIndexCountZero(...args),
|
|
2119
|
+
readAverageFromBuckets: (...args) => readAverageFromBuckets(...args),
|
|
2120
|
+
readCountFieldFromBuckets: (...args) => readCountFieldFromBuckets(...args),
|
|
2121
|
+
readCountFromBuckets: (...args) => readCountFromBuckets(...args),
|
|
2122
|
+
readExtremaFromBuckets: (...args) => readExtremaFromBuckets(...args),
|
|
2123
|
+
readRankAt: (...args) => readRankAt(...args),
|
|
2124
|
+
readRankCount: (...args) => readRankCount(...args),
|
|
2125
|
+
readRankIndexOf: (...args) => readRankIndexOf(...args),
|
|
2126
|
+
readRankMax: (...args) => readRankMax(...args),
|
|
2127
|
+
readRankMin: (...args) => readRankMin(...args),
|
|
2128
|
+
readRankPaginate: (...args) => readRankPaginate(...args),
|
|
2129
|
+
readRankRandom: (...args) => readRankRandom(...args),
|
|
2130
|
+
readRankSum: (...args) => readRankSum(...args),
|
|
2131
|
+
readSumFromBuckets: (...args) => readSumFromBuckets(...args)
|
|
2132
|
+
}
|
|
2133
|
+
});
|
|
2134
|
+
|
|
2135
|
+
//#endregion
|
|
2136
|
+
export { aggregateCapability };
|