@aglyn/plugins-data 1.0.0-beta.219 → 1.0.0-beta.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +10 -10
- package/src/lib/constants/bundle-common.d.ts +2 -0
- package/src/lib/constants/bundle-common.js +1 -0
- package/src/lib/constants/bundle-common.js.map +1 -1
- package/src/lib/declarations.console-server.d.ts +47 -0
- package/src/lib/declarations.console-server.js +74 -0
- package/src/lib/declarations.console-server.js.map +1 -0
- package/src/lib/declarations.server.d.ts +4 -0
- package/src/lib/declarations.server.js +11 -1
- package/src/lib/declarations.server.js.map +1 -1
- package/src/lib/model/dataset-csv.d.ts +36 -0
- package/src/lib/model/dataset-csv.js +85 -0
- package/src/lib/model/dataset-csv.js.map +1 -0
- package/src/lib/model/dataset-io.d.ts +8 -7
- package/src/lib/model/dataset-io.js +9 -8
- package/src/lib/model/dataset-io.js.map +1 -1
- package/src/lib/model/dataset-record-list.d.ts +40 -0
- package/src/lib/model/dataset-record-list.js +61 -0
- package/src/lib/model/dataset-record-list.js.map +1 -0
- package/src/lib/model/dataset-record.d.ts +33 -0
- package/src/lib/model/dataset-record.js +68 -0
- package/src/lib/model/dataset-record.js.map +1 -0
- package/src/lib/plugin.js +3 -0
- package/src/lib/plugin.js.map +1 -1
- package/src/lib/server/announce-dataset-records.d.ts +37 -0
- package/src/lib/server/announce-dataset-records.js +54 -0
- package/src/lib/server/announce-dataset-records.js.map +1 -0
- package/src/lib/server/api-v1/datasets.d.ts +18 -0
- package/src/lib/server/api-v1/datasets.js +646 -0
- package/src/lib/server/api-v1/datasets.js.map +1 -0
- package/src/lib/server/api-v1/openapi.d.ts +26 -0
- package/src/lib/server/api-v1/openapi.js +260 -0
- package/src/lib/server/api-v1/openapi.js.map +1 -0
- package/src/lib/server/api-v1/usage.d.ts +26 -0
- package/src/lib/server/api-v1/usage.js +33 -0
- package/src/lib/server/api-v1/usage.js.map +1 -0
- package/src/lib/server/custom-field-types.d.ts +39 -0
- package/src/lib/server/custom-field-types.js +41 -0
- package/src/lib/server/custom-field-types.js.map +1 -0
- package/src/lib/server/dataset-figures.d.ts +61 -0
- package/src/lib/server/dataset-figures.js +413 -0
- package/src/lib/server/dataset-figures.js.map +1 -0
- package/src/lib/server/dataset-record-index.d.ts +18 -0
- package/src/lib/server/dataset-record-index.js +62 -0
- package/src/lib/server/dataset-record-index.js.map +1 -0
- package/src/lib/server/dataset-storage-meter.d.ts +29 -0
- package/src/lib/server/dataset-storage-meter.js +82 -0
- package/src/lib/server/dataset-storage-meter.js.map +1 -0
- package/src/lib/server/datasets-export-route.d.ts +57 -0
- package/src/lib/server/datasets-export-route.js +229 -0
- package/src/lib/server/datasets-export-route.js.map +1 -0
- package/src/lib/server/datasets-route.d.ts +38 -0
- package/src/lib/server/datasets-route.js +614 -0
- package/src/lib/server/datasets-route.js.map +1 -0
- package/src/lib/server.d.ts +23 -0
- package/src/lib/server.js +54 -0
- package/src/lib/server.js.map +1 -0
- package/src/lib/site-bundle/datasets-site-bundle.server.js +2 -15
- package/src/lib/site-bundle/datasets-site-bundle.server.js.map +1 -1
- package/src/lib/usage-axes.d.ts +24 -0
- package/src/lib/usage-axes.js +32 -0
- package/src/lib/usage-axes.js.map +1 -0
|
@@ -0,0 +1,614 @@
|
|
|
1
|
+
import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
2
|
+
/**
|
|
3
|
+
* @license
|
|
4
|
+
* Copyright 2026 Aglyn LLC
|
|
5
|
+
*
|
|
6
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
* you may not use this file except in compliance with the License.
|
|
8
|
+
* You may obtain a copy of the License at
|
|
9
|
+
*
|
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
*
|
|
12
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
* See the License for the specific language governing permissions and
|
|
16
|
+
* limitations under the License.
|
|
17
|
+
*/ import { defaultScopeForNewResource, newResourceScopeFields, pluginRequestFromWeb } from "@aglyn/aglyn/server";
|
|
18
|
+
import { checkDatasetQuota, checkEntitlement, checkQuota, coerceDocumentValues, createResourceUid, datasetIntegrityFields, effectiveDatasetModel, ensureDeclaredCustomFieldTypes, memberCanSee, validateDocument } from "@aglyn/aglyn/server";
|
|
19
|
+
import { emailUnverifiedResponse, firebaseAdmin, dataStorageRefusal, isImpersonationSession, isServerReleaseFlagOnForOrg, lockdownRefusal, memberHasOrgPermission, resolveOrgIdForHost, resolveOrgMembership } from "@aglyn/tenant-data-admin";
|
|
20
|
+
import { isRefusedIdToken } from "@aglyn/tenant-data-admin/server/id-token-refusal";
|
|
21
|
+
import { Timestamp } from "firebase-admin/firestore";
|
|
22
|
+
import { announceDatasetRecords as announceDatasetChange } from "./announce-dataset-records.js";
|
|
23
|
+
/**
|
|
24
|
+
* `dataStorageMbPerOrg` for this route, rendered as the console's 403.
|
|
25
|
+
*
|
|
26
|
+
* The VERDICT moved to `utils/server/data-storage-gate` (AGL-2253) — it was
|
|
27
|
+
* defined here, so `/v1` and the tenant form path, which write dataset bytes
|
|
28
|
+
* through their own code, enforced nothing. Only the console wording is left
|
|
29
|
+
* behind, because the other two callers answer different shapes.
|
|
30
|
+
*/ async function refuseIfDataStorageBlocked(org, orgRef) {
|
|
31
|
+
const refusal = await dataStorageRefusal(org, orgRef);
|
|
32
|
+
if (!refusal) return null;
|
|
33
|
+
return Response.json({
|
|
34
|
+
error: refusal.basis === 'always' ? 'Dataset storage is not included on this plan — upgrade in Billing' : `Dataset storage limit reached (${refusal.includedMb} MB) — ` + 'upgrade in Billing'
|
|
35
|
+
}, {
|
|
36
|
+
status: 403
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
/** Roles allowed to create org data — mirrors rules' canWriteOrgData(). */ const WRITER_ROLES = new Set([
|
|
40
|
+
'owner',
|
|
41
|
+
'admin',
|
|
42
|
+
'editor'
|
|
43
|
+
]);
|
|
44
|
+
/**
|
|
45
|
+
* Rows written per import transaction.
|
|
46
|
+
*
|
|
47
|
+
* Firestore allows at most 500 writes in one transaction, and this is the
|
|
48
|
+
* chunk size the route already used for its batches, so nothing about the
|
|
49
|
+
* shape of an import changed when the batches became transactions (AGL-2371).
|
|
50
|
+
*/ const IMPORT_CHUNK = 400;
|
|
51
|
+
/**
|
|
52
|
+
* Dataset/record creation API (AGL-473): creates moved out of the client
|
|
53
|
+
* SDK so quotas and entitlements are enforced server-side — Firestore
|
|
54
|
+
* rules deny client-side `create` on `orgs/{orgId}/datasets/**` (updates
|
|
55
|
+
* and deletes stay client-direct, they don't consume quota). Actions:
|
|
56
|
+
*
|
|
57
|
+
* - `create-dataset`: `dataStore` entitlement + `checkDatasetQuota`
|
|
58
|
+
* (addon-aware, org-scoped). An optional `hostId` names the site the
|
|
59
|
+
* dataset is created from, which the org's Default sharing can narrow it
|
|
60
|
+
* to.
|
|
61
|
+
* - `create-record`: `recordsPerDataset` quota; values are re-coerced
|
|
62
|
+
* and re-validated against the dataset's model server-side.
|
|
63
|
+
* - `import-records`: batch create with the whole batch fitting the cap.
|
|
64
|
+
*
|
|
65
|
+
* Since AGL-2163 both record actions ALSO enforce `checkDataStorageQuota`.
|
|
66
|
+
* See {@link refuseIfDataStorageBlocked}: that check's `allowed` field had no
|
|
67
|
+
* reader anywhere in the platform while its docblock said free "hard-blocks
|
|
68
|
+
* at the included size", so the byte band was documentation. `recordsPerDataset`
|
|
69
|
+
* counts ROWS and does not bound their size, so it was never the same limit.
|
|
70
|
+
*/ export const datasetsHandler = async (request)=>{
|
|
71
|
+
var _headers_authorization, _ref, _ref1;
|
|
72
|
+
const { method, body, headers: rawHeaders } = await pluginRequestFromWeb(request);
|
|
73
|
+
const headers = rawHeaders;
|
|
74
|
+
if (method !== 'POST') {
|
|
75
|
+
return Response.json({
|
|
76
|
+
error: 'Method not allowed'
|
|
77
|
+
}, {
|
|
78
|
+
status: 405
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
const authorization = (_headers_authorization = headers.authorization) != null ? _headers_authorization : '';
|
|
82
|
+
const idToken = authorization.startsWith('Bearer ') ? authorization.slice('Bearer '.length) : undefined;
|
|
83
|
+
if (!idToken) return Response.json({
|
|
84
|
+
error: 'Unauthenticated'
|
|
85
|
+
}, {
|
|
86
|
+
status: 401
|
|
87
|
+
});
|
|
88
|
+
const orgId = String((_ref = body == null ? void 0 : body.orgId) != null ? _ref : '');
|
|
89
|
+
const action = String((_ref1 = body == null ? void 0 : body.action) != null ? _ref1 : '');
|
|
90
|
+
if (!orgId) return Response.json({
|
|
91
|
+
error: 'Missing orgId'
|
|
92
|
+
}, {
|
|
93
|
+
status: 400
|
|
94
|
+
});
|
|
95
|
+
try {
|
|
96
|
+
const decoded = await firebaseAdmin.app().auth().verifyIdToken(idToken);
|
|
97
|
+
if (!decoded.email_verified && !isImpersonationSession(decoded)) {
|
|
98
|
+
return emailUnverifiedResponse();
|
|
99
|
+
}
|
|
100
|
+
const membership = await resolveOrgMembership(decoded.uid, orgId);
|
|
101
|
+
const member = membership == null ? void 0 : membership.member;
|
|
102
|
+
if (!member || !WRITER_ROLES.has(String(member.role))) {
|
|
103
|
+
return Response.json({
|
|
104
|
+
error: 'Editing org data requires the editor role'
|
|
105
|
+
}, {
|
|
106
|
+
status: 403
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
/*==========================================
|
|
110
|
+
* AND the granular permission (AGL-2444).
|
|
111
|
+
*
|
|
112
|
+
* `data.manage` had ZERO consumers: an owner could build a custom role
|
|
113
|
+
* with "Manage data" unticked, assign it, and the member kept creating,
|
|
114
|
+
* editing and deleting datasets — because this gate read the raw role,
|
|
115
|
+
* exactly the defect AGL-2350 fixed one route over on `hosts.create`.
|
|
116
|
+
*
|
|
117
|
+
* Identical for the built-in roles — `data.manage` is on for owner,
|
|
118
|
+
* admin and editor, which is `WRITER_ROLES` — so nobody's access moves.
|
|
119
|
+
* What it adds is that a custom role or a per-member override is now
|
|
120
|
+
* honoured, which is the narrowing the docs sell.
|
|
121
|
+
*
|
|
122
|
+
* The role check is kept ALONGSIDE it rather than replaced. It mirrors
|
|
123
|
+
* the rules' `canWriteOrgData()`, and a route that disagreed with the
|
|
124
|
+
* rules would let one door open while the other stayed shut.
|
|
125
|
+
*=========================================*/ if (decoded['staff'] !== true && !await memberHasOrgPermission(orgId, member, 'data.manage')) {
|
|
126
|
+
return Response.json({
|
|
127
|
+
error: 'Your organization role does not allow editing organization data'
|
|
128
|
+
}, {
|
|
129
|
+
status: 403
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
// Release gate (AGL-1653). `<FeatureGate flag="release_data_store">` on
|
|
133
|
+
// the org Data page was the only gate, so org-level dataset
|
|
134
|
+
// administration kept working with the flag off — the milder sibling of
|
|
135
|
+
// the add-on store leak above it in the same issue. Same staff bypass as
|
|
136
|
+
// `FeatureGate`'s `visible`, same 404: released-off means absent.
|
|
137
|
+
if (decoded['staff'] !== true && !await isServerReleaseFlagOnForOrg('release_data_store', orgId)) {
|
|
138
|
+
return Response.json({
|
|
139
|
+
error: 'Not available'
|
|
140
|
+
}, {
|
|
141
|
+
status: 404
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
const firestore = firebaseAdmin.app().firestore();
|
|
145
|
+
const orgRef = firestore.collection('orgs').doc(orgId);
|
|
146
|
+
const orgSnapshot = await orgRef.get();
|
|
147
|
+
if (!orgSnapshot.exists) {
|
|
148
|
+
return Response.json({
|
|
149
|
+
error: 'Unknown organization'
|
|
150
|
+
}, {
|
|
151
|
+
status: 404
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
const org = orgSnapshot.data();
|
|
155
|
+
// Lockdown verdict (AGL-1506): subsumes the bare `orgSuspended`
|
|
156
|
+
// projection check this route used to make — the org doc is read just
|
|
157
|
+
// above anyway, so the full verdict (platform/org/user scopes, staff
|
|
158
|
+
// bypass, distinct 423 body) costs no extra org read here.
|
|
159
|
+
const locked = await lockdownRefusal({
|
|
160
|
+
request,
|
|
161
|
+
staff: decoded['staff'] === true,
|
|
162
|
+
uid: decoded.uid,
|
|
163
|
+
org
|
|
164
|
+
});
|
|
165
|
+
if (locked) return locked;
|
|
166
|
+
if (action === 'create-dataset') {
|
|
167
|
+
var _ref2, _ref3;
|
|
168
|
+
if (!checkEntitlement(org, 'dataStore')) {
|
|
169
|
+
return Response.json({
|
|
170
|
+
error: 'Datasets require a Starter plan or higher'
|
|
171
|
+
}, {
|
|
172
|
+
status: 403
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
const displayName = String((_ref2 = body == null ? void 0 : body.displayName) != null ? _ref2 : '').trim().slice(0, 120);
|
|
176
|
+
const fields = Array.isArray(body == null ? void 0 : body.fields) ? body.fields.map((field)=>String(field)).slice(0, 100) : [];
|
|
177
|
+
if (!displayName || fields.length === 0) {
|
|
178
|
+
return Response.json({
|
|
179
|
+
error: 'Missing displayName or fields'
|
|
180
|
+
}, {
|
|
181
|
+
status: 400
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* The site the dataset is created from — null on the organization Data
|
|
186
|
+
* page, which has none.
|
|
187
|
+
*
|
|
188
|
+
* It reaches the document only through the org's Default sharing: with
|
|
189
|
+
* `defaultResourceScope: 'host'` the new dataset is visible to this site
|
|
190
|
+
* alone. That makes it a client claim deciding who can see the result,
|
|
191
|
+
* so it is honored only when it holds on both counts:
|
|
192
|
+
*
|
|
193
|
+
* * The site is one of THIS org's. `hostIndex` is the mirror the Data
|
|
194
|
+
* card resolved `orgId` from in the first place, so a create made
|
|
195
|
+
* from a real site page cannot disagree with it. Another org's site
|
|
196
|
+
* would stamp a token that no site and no scoped member of this org
|
|
197
|
+
* holds: a dataset only the org's admins could ever find.
|
|
198
|
+
* * The caller's own access reaches the scope it produces. A scoped
|
|
199
|
+
* member naming a site outside their access would create a dataset
|
|
200
|
+
* they cannot see afterwards, which is the create the rules'
|
|
201
|
+
* `canCreateScoped()` refuses on the collections clients create.
|
|
202
|
+
*
|
|
203
|
+
* Both are refused rather than widened to All sites. Widening is the
|
|
204
|
+
* direction that shows a dataset to sites nobody chose.
|
|
205
|
+
*/ const hostId = String((_ref3 = body == null ? void 0 : body.hostId) != null ? _ref3 : '').trim() || null;
|
|
206
|
+
if (hostId && (hostId.includes('/') || await resolveOrgIdForHost(hostId) !== orgId)) {
|
|
207
|
+
return Response.json({
|
|
208
|
+
error: 'That site is not one of this organization’s'
|
|
209
|
+
}, {
|
|
210
|
+
status: 400
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
// Honors the org's `defaultResourceScope` (AGL-1048), falling back to
|
|
214
|
+
// org-wide — which is also the only answer with no site in context.
|
|
215
|
+
const visibleTo = defaultScopeForNewResource({
|
|
216
|
+
defaultResourceScope: org == null ? void 0 : org.defaultResourceScope,
|
|
217
|
+
hostId
|
|
218
|
+
});
|
|
219
|
+
if (hostId && decoded['staff'] !== true && !memberCanSee(member, visibleTo)) {
|
|
220
|
+
return Response.json({
|
|
221
|
+
error: 'Your access to this organization does not reach that site'
|
|
222
|
+
}, {
|
|
223
|
+
status: 403
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
const datasetsRef = orgRef.collection('datasets');
|
|
227
|
+
const overDatasetQuota = (quota)=>quota.upgradeRequired ? `Dataset limit reached (${quota.limit}) — upgrade in Billing` : `Dataset limit reached (${quota.limit}) — add extra datasets ` + `for $${quota.addonPriceUsd}/mo each or upgrade in Billing`;
|
|
228
|
+
// The fast refusal. NOT the enforcement point — that is the transaction
|
|
229
|
+
// below — but it answers a caller who is simply over their limit before
|
|
230
|
+
// the request builds anything.
|
|
231
|
+
const quota = checkDatasetQuota(org, (await datasetsRef.count().get()).data().count);
|
|
232
|
+
if (!quota.allowed) {
|
|
233
|
+
return Response.json({
|
|
234
|
+
error: overDatasetQuota(quota)
|
|
235
|
+
}, {
|
|
236
|
+
status: 403
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
const id = createResourceUid();
|
|
240
|
+
// The model rides from the console (deriveModelFromFields output or
|
|
241
|
+
// the join-collection template). `model` is independent of `fields`,
|
|
242
|
+
// so cap its serialized size explicitly (a legit model is well under
|
|
243
|
+
// this — 64 KB covers hundreds of typed fields).
|
|
244
|
+
const model = (body == null ? void 0 : body.model) && typeof body.model === 'object' ? body.model : null;
|
|
245
|
+
if (model && JSON.stringify(model).length > 64 * 1024) {
|
|
246
|
+
return Response.json({
|
|
247
|
+
error: 'Dataset model too large'
|
|
248
|
+
}, {
|
|
249
|
+
status: 413
|
|
250
|
+
});
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* THE ENFORCEMENT POINT (AGL-2371, the AGL-2231 treatment): the count,
|
|
254
|
+
* the decision and the create in ONE transaction.
|
|
255
|
+
*
|
|
256
|
+
* The check above counted, then the request went on awaiting, and only
|
|
257
|
+
* then created — and every await is a yield, so N concurrent creates
|
|
258
|
+
* each read the same pre-count, each found room, and each landed.
|
|
259
|
+
* Nothing re-counts afterwards, so the extra datasets were permanent and
|
|
260
|
+
* billed nothing. `tx.get` on the aggregate takes a pessimistic lock on
|
|
261
|
+
* the documents the query matched, so the loser of a race retries,
|
|
262
|
+
* re-reads the higher count and is refused.
|
|
263
|
+
*
|
|
264
|
+
* The ORG document is deliberately read outside: the plan and the
|
|
265
|
+
* purchased add-ons are the other input to this decision, but neither is
|
|
266
|
+
* client-writable, so locking the org doc on every dataset create would
|
|
267
|
+
* buy contention rather than correctness. That is the AGL-2369 lesson
|
|
268
|
+
* applied, not skipped — there the routing map *was* client-writable and
|
|
269
|
+
* *did* change the count.
|
|
270
|
+
*
|
|
271
|
+
* The refusal comes back as data and is rendered outside: building a
|
|
272
|
+
* response inside a body that can run several times reads as if the
|
|
273
|
+
* transaction were a place effects happen.
|
|
274
|
+
*/ const refusal = await firestore.runTransaction(async (tx)=>{
|
|
275
|
+
const live = (await tx.get(datasetsRef.count())).data().count;
|
|
276
|
+
const authoritative = checkDatasetQuota(org, live);
|
|
277
|
+
if (!authoritative.allowed) return overDatasetQuota(authoritative);
|
|
278
|
+
tx.create(datasetsRef.doc(id), _extends({
|
|
279
|
+
displayName,
|
|
280
|
+
fields
|
|
281
|
+
}, model ? {
|
|
282
|
+
model
|
|
283
|
+
} : {}, newResourceScopeFields(visibleTo), {
|
|
284
|
+
createdAt: Timestamp.now()
|
|
285
|
+
}));
|
|
286
|
+
return null;
|
|
287
|
+
});
|
|
288
|
+
if (refusal) return Response.json({
|
|
289
|
+
error: refusal
|
|
290
|
+
}, {
|
|
291
|
+
status: 403
|
|
292
|
+
});
|
|
293
|
+
return Response.json({
|
|
294
|
+
ok: true,
|
|
295
|
+
id
|
|
296
|
+
}, {
|
|
297
|
+
status: 200
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
if (action === 'create-record' || action === 'import-records' || action === 'announce-records') {
|
|
301
|
+
var _ref4;
|
|
302
|
+
var _datasetSnapshot_data;
|
|
303
|
+
const datasetId = String((_ref4 = body == null ? void 0 : body.datasetId) != null ? _ref4 : '');
|
|
304
|
+
if (!datasetId) {
|
|
305
|
+
return Response.json({
|
|
306
|
+
error: 'Missing datasetId'
|
|
307
|
+
}, {
|
|
308
|
+
status: 400
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
const datasetRef = orgRef.collection('datasets').doc(datasetId);
|
|
312
|
+
const datasetSnapshot = await datasetRef.get();
|
|
313
|
+
if (!datasetSnapshot.exists) {
|
|
314
|
+
return Response.json({
|
|
315
|
+
error: 'Unknown dataset'
|
|
316
|
+
}, {
|
|
317
|
+
status: 404
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* The caller has to be able to SEE the dataset they add rows to.
|
|
322
|
+
*
|
|
323
|
+
* The role and `data.manage` gates above are about the org, and a member
|
|
324
|
+
* scoped to one site passes both. The rules hold every record update and
|
|
325
|
+
* delete to the parent dataset being visible (`parentDatasetVisible()`),
|
|
326
|
+
* and refuse client record creates outright, so this route is the one
|
|
327
|
+
* place a member's create can be held to the same condition. Without it,
|
|
328
|
+
* a collaborator on one client's site could add rows, by id alone, to an
|
|
329
|
+
* internal dataset shared only with other sites.
|
|
330
|
+
*
|
|
331
|
+
* The Admin SDK evaluates no rules, so `memberCanSee` IS the enforcement
|
|
332
|
+
* here, as on the export route beside this one. Answered with the same
|
|
333
|
+
* 404 as a dataset that does not exist: whether one outside the caller's
|
|
334
|
+
* reach exists is not theirs to learn either.
|
|
335
|
+
*/ if (decoded['staff'] !== true && !memberCanSee(member, (_datasetSnapshot_data = datasetSnapshot.data()) == null ? void 0 : _datasetSnapshot_data.visibleTo)) {
|
|
336
|
+
return Response.json({
|
|
337
|
+
error: 'Unknown dataset'
|
|
338
|
+
}, {
|
|
339
|
+
status: 404
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* A record the BROWSER wrote, announcing itself (AGL-3113).
|
|
344
|
+
*
|
|
345
|
+
* Record edits and deletes stay client-direct on purpose — they consume
|
|
346
|
+
* no quota, so AGL-473 left them out of the server leg — which also left
|
|
347
|
+
* them with nothing to announce from. They are the same staleness as a
|
|
348
|
+
* create: a page listing the rows goes on showing the old one.
|
|
349
|
+
*
|
|
350
|
+
* A drop, and only a drop. It moves no pointer, writes nothing and
|
|
351
|
+
* registers no route, so it grants nothing the rules withhold — and it
|
|
352
|
+
* is gated by the same org membership and the same `memberCanSee` the
|
|
353
|
+
* create is, which is what stops a collaborator on one site from
|
|
354
|
+
* learning that another site's dataset exists.
|
|
355
|
+
*/ if (action === 'announce-records') {
|
|
356
|
+
const announced = await announceDatasetChange({
|
|
357
|
+
firestore,
|
|
358
|
+
orgId,
|
|
359
|
+
datasetId
|
|
360
|
+
});
|
|
361
|
+
return Response.json({
|
|
362
|
+
ok: true,
|
|
363
|
+
announced
|
|
364
|
+
}, {
|
|
365
|
+
status: 200
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
const model = effectiveDatasetModel(datasetSnapshot.data());
|
|
369
|
+
// Before either validation below: a plugin's field validator only runs
|
|
370
|
+
// once its plugin's server entry has registered it.
|
|
371
|
+
await ensureDeclaredCustomFieldTypes(model);
|
|
372
|
+
const recordsRef = datasetRef.collection('records');
|
|
373
|
+
const recordCount = (await recordsRef.count().get()).data().count;
|
|
374
|
+
const overRecordQuota = (limit)=>`Record limit reached (${limit}) — upgrade in Billing`;
|
|
375
|
+
if (action === 'create-record') {
|
|
376
|
+
var _ref5;
|
|
377
|
+
const coerced = coerceDocumentValues(model, (_ref5 = body == null ? void 0 : body.values) != null ? _ref5 : {});
|
|
378
|
+
const errors = validateDocument(model, coerced);
|
|
379
|
+
if (Object.keys(errors).length) {
|
|
380
|
+
return Response.json({
|
|
381
|
+
error: 'Record failed validation',
|
|
382
|
+
errors
|
|
383
|
+
}, {
|
|
384
|
+
status: 400
|
|
385
|
+
});
|
|
386
|
+
}
|
|
387
|
+
// The fast refusal. NOT the enforcement point — that is the
|
|
388
|
+
// transaction below — but it keeps the refusal ORDER on this route
|
|
389
|
+
// intact: a caller over BOTH the row cap and the byte band is told
|
|
390
|
+
// about the rows, as they always were.
|
|
391
|
+
const quota = checkQuota(org, 'recordsPerDataset', recordCount);
|
|
392
|
+
if (!quota.allowed) {
|
|
393
|
+
return Response.json({
|
|
394
|
+
error: overRecordQuota(quota.limit)
|
|
395
|
+
}, {
|
|
396
|
+
status: 403
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
// Bytes, not rows (AGL-2163). Before the write, like every other
|
|
400
|
+
// quota on this route.
|
|
401
|
+
const storageRefusal = await refuseIfDataStorageBlocked(org, orgRef);
|
|
402
|
+
if (storageRefusal) return storageRefusal;
|
|
403
|
+
const id = createResourceUid();
|
|
404
|
+
/**
|
|
405
|
+
* THE ENFORCEMENT POINT (AGL-2371, the AGL-2231 treatment): the count,
|
|
406
|
+
* the decision and the create in ONE transaction.
|
|
407
|
+
*
|
|
408
|
+
* The check above counted, then `refuseIfDataStorageBlocked` awaited,
|
|
409
|
+
* and only then did the row land. Every await is a yield, so N
|
|
410
|
+
* concurrent creates each read the same pre-count, each found room,
|
|
411
|
+
* and each landed — permanently, since nothing re-counts afterwards. A
|
|
412
|
+
* free-with-rows-granted org's thousand rows became twenty thousand by
|
|
413
|
+
* sending twenty requests at once.
|
|
414
|
+
*
|
|
415
|
+
* `tx.get` on the aggregate takes a pessimistic lock on the documents
|
|
416
|
+
* the query matched, so the loser retries, re-reads the higher count
|
|
417
|
+
* and is refused. `order` is taken from the transaction's own count
|
|
418
|
+
* for the same reason: the pre-count is what made two concurrent rows
|
|
419
|
+
* share an order.
|
|
420
|
+
*
|
|
421
|
+
* The BYTE gate stays outside. It is a different quota with its own
|
|
422
|
+
* reads, it is not a function of the rows this transaction writes, and
|
|
423
|
+
* re-running it on every retry attempt would put a multi-document read
|
|
424
|
+
* inside a lock.
|
|
425
|
+
*/ const refusal = await firestore.runTransaction(async (tx)=>{
|
|
426
|
+
const live = (await tx.get(recordsRef.count())).data().count;
|
|
427
|
+
const authoritative = checkQuota(org, 'recordsPerDataset', live);
|
|
428
|
+
if (!authoritative.allowed) return overRecordQuota(authoritative.limit);
|
|
429
|
+
tx.create(recordsRef.doc(id), _extends({
|
|
430
|
+
values: coerced
|
|
431
|
+
}, datasetIntegrityFields(model, coerced), {
|
|
432
|
+
order: live,
|
|
433
|
+
createdAt: Timestamp.now(),
|
|
434
|
+
updatedAt: Timestamp.now()
|
|
435
|
+
}));
|
|
436
|
+
return null;
|
|
437
|
+
});
|
|
438
|
+
if (refusal) return Response.json({
|
|
439
|
+
error: refusal
|
|
440
|
+
}, {
|
|
441
|
+
status: 403
|
|
442
|
+
});
|
|
443
|
+
// The record is stored; now the pages that repeat over it are told
|
|
444
|
+
// (AGL-3113). Awaited so the site is already serving the new row by the
|
|
445
|
+
// time the editor's save returns, and reported so a refused drop can
|
|
446
|
+
// say the live pages are behind rather than leaving it to be noticed.
|
|
447
|
+
const announced = await announceDatasetChange({
|
|
448
|
+
firestore,
|
|
449
|
+
orgId,
|
|
450
|
+
datasetId
|
|
451
|
+
});
|
|
452
|
+
return Response.json({
|
|
453
|
+
ok: true,
|
|
454
|
+
id,
|
|
455
|
+
announced
|
|
456
|
+
}, {
|
|
457
|
+
status: 200
|
|
458
|
+
});
|
|
459
|
+
}
|
|
460
|
+
// import-records: the console sends only the NEW rows (updates to
|
|
461
|
+
// existing ids stay client-direct — they don't consume quota).
|
|
462
|
+
const rows = Array.isArray(body == null ? void 0 : body.records) ? body.records : [];
|
|
463
|
+
if (rows.length === 0) {
|
|
464
|
+
return Response.json({
|
|
465
|
+
error: 'No records to import'
|
|
466
|
+
}, {
|
|
467
|
+
status: 400
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
const overImportQuota = (limit, needed)=>`This import needs ${needed} record slots — your plan ` + `allows ${limit} per dataset. See Billing to upgrade.`;
|
|
471
|
+
// The fast refusal, on the whole import. NOT the enforcement point —
|
|
472
|
+
// that is the per-chunk transaction below — but it is what answers an
|
|
473
|
+
// import that never fitted, before a single row is written.
|
|
474
|
+
const quota = checkQuota(org, 'recordsPerDataset', recordCount + rows.length - 1);
|
|
475
|
+
if (!quota.allowed) {
|
|
476
|
+
return Response.json({
|
|
477
|
+
error: overImportQuota(quota.limit, rows.length)
|
|
478
|
+
}, {
|
|
479
|
+
status: 403
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
// Bytes, not rows (AGL-2163). An import is the path that moves the
|
|
483
|
+
// most bytes in one request, so leaving it out would have left the
|
|
484
|
+
// enforcement exactly where the customer does not meet it.
|
|
485
|
+
const importStorageRefusal = await refuseIfDataStorageBlocked(org, orgRef);
|
|
486
|
+
if (importStorageRefusal) return importStorageRefusal;
|
|
487
|
+
const prepared = rows.map((row, index)=>{
|
|
488
|
+
var _ref;
|
|
489
|
+
const coerced = coerceDocumentValues(model, (_ref = row == null ? void 0 : row.values) != null ? _ref : {});
|
|
490
|
+
const errors = validateDocument(model, coerced);
|
|
491
|
+
return {
|
|
492
|
+
index,
|
|
493
|
+
coerced,
|
|
494
|
+
valid: Object.keys(errors).length === 0
|
|
495
|
+
};
|
|
496
|
+
});
|
|
497
|
+
const invalid = prepared.filter((row)=>!row.valid);
|
|
498
|
+
if (invalid.length) {
|
|
499
|
+
return Response.json({
|
|
500
|
+
error: `${invalid.length} rows failed validation`,
|
|
501
|
+
rows: invalid.map((row)=>row.index)
|
|
502
|
+
}, {
|
|
503
|
+
status: 400
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
/**
|
|
507
|
+
* THE ENFORCEMENT POINT (AGL-2371): each chunk counts, decides and
|
|
508
|
+
* writes inside ONE transaction.
|
|
509
|
+
*
|
|
510
|
+
* The import used to count once, decide once, and then commit a series
|
|
511
|
+
* of `WriteBatch`es. A batch is atomic but NOT conditional on a read
|
|
512
|
+
* taken before it, so two concurrent imports each priced themselves off
|
|
513
|
+
* the same pre-count and both landed in full — the create leg's defect,
|
|
514
|
+
* multiplied by the batch size.
|
|
515
|
+
*
|
|
516
|
+
* It cannot be ONE transaction for every import the way the single-row
|
|
517
|
+
* create can: `recordsPerDataset` runs to a million, and a transaction
|
|
518
|
+
* carries at most 500 writes. So the chunk is the unit, and each chunk's
|
|
519
|
+
* count is re-read under the lock and re-checked against the plan. The
|
|
520
|
+
* cap therefore holds ABSOLUTELY at every size — what a race can cost is
|
|
521
|
+
* a partial import, not an overshoot, and a partial import was already
|
|
522
|
+
* possible the moment a mid-import batch could fail.
|
|
523
|
+
*
|
|
524
|
+
* The ids are minted before the transaction so a retry re-creates the
|
|
525
|
+
* SAME documents rather than a second copy of the chunk.
|
|
526
|
+
*
|
|
527
|
+
* This is not AGL-2370's problem in miniature. There the whole bundle
|
|
528
|
+
* must land or none of it, so chunking cannot be the unit and a lease is
|
|
529
|
+
* needed; here rows are independent and "fewer rows than asked for" is a
|
|
530
|
+
* reportable outcome.
|
|
531
|
+
*/ const ids = [];
|
|
532
|
+
let refusedAt = null;
|
|
533
|
+
for(let start = 0; start < prepared.length && refusedAt === null; start += IMPORT_CHUNK){
|
|
534
|
+
const chunk = prepared.slice(start, start + IMPORT_CHUNK);
|
|
535
|
+
const chunkIds = chunk.map(()=>createResourceUid());
|
|
536
|
+
refusedAt = await firestore.runTransaction(async (tx)=>{
|
|
537
|
+
const live = (await tx.get(recordsRef.count())).data().count;
|
|
538
|
+
const authoritative = checkQuota(org, 'recordsPerDataset', live + chunk.length - 1);
|
|
539
|
+
if (!authoritative.allowed) {
|
|
540
|
+
return overImportQuota(authoritative.limit, rows.length);
|
|
541
|
+
}
|
|
542
|
+
chunk.forEach((row, offset)=>{
|
|
543
|
+
tx.create(recordsRef.doc(chunkIds[offset]), _extends({
|
|
544
|
+
values: row.coerced
|
|
545
|
+
}, datasetIntegrityFields(model, row.coerced), {
|
|
546
|
+
order: live + offset,
|
|
547
|
+
createdAt: Timestamp.now(),
|
|
548
|
+
updatedAt: Timestamp.now()
|
|
549
|
+
}));
|
|
550
|
+
});
|
|
551
|
+
return null;
|
|
552
|
+
});
|
|
553
|
+
if (refusedAt === null) ids.push(...chunkIds);
|
|
554
|
+
}
|
|
555
|
+
/**
|
|
556
|
+
* ONE announce for the whole import, after the loop (AGL-3113).
|
|
557
|
+
*
|
|
558
|
+
* Not per chunk and certainly not per row: a thousand rows arrive as one
|
|
559
|
+
* import and make the same pages stale once, so the pages are worked out
|
|
560
|
+
* once and dropped once. The refusal path announces too — rows that
|
|
561
|
+
* landed before the quota bit are live records, and leaving them out of
|
|
562
|
+
* the refresh would make a partial import look like no import at all.
|
|
563
|
+
*/ if (ids.length) {
|
|
564
|
+
await announceDatasetChange({
|
|
565
|
+
firestore,
|
|
566
|
+
orgId,
|
|
567
|
+
datasetId
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
if (refusedAt !== null) {
|
|
571
|
+
// What landed is reported alongside the refusal. Rows are independent,
|
|
572
|
+
// so silently discarding the created ids would tell the caller nothing
|
|
573
|
+
// was written while the dataset says otherwise.
|
|
574
|
+
return Response.json({
|
|
575
|
+
error: refusedAt,
|
|
576
|
+
ids,
|
|
577
|
+
created: ids.length
|
|
578
|
+
}, {
|
|
579
|
+
status: 403
|
|
580
|
+
});
|
|
581
|
+
}
|
|
582
|
+
return Response.json({
|
|
583
|
+
ok: true,
|
|
584
|
+
ids,
|
|
585
|
+
created: ids.length
|
|
586
|
+
}, {
|
|
587
|
+
status: 200
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
return Response.json({
|
|
591
|
+
error: 'Unknown action'
|
|
592
|
+
}, {
|
|
593
|
+
status: 400
|
|
594
|
+
});
|
|
595
|
+
} catch (error) {
|
|
596
|
+
// A refused credential is a 401, not a fault of ours (AGL-1993); anything
|
|
597
|
+
// else keeps the answer below.
|
|
598
|
+
if (isRefusedIdToken(error)) {
|
|
599
|
+
return Response.json({
|
|
600
|
+
error: 'Unauthenticated'
|
|
601
|
+
}, {
|
|
602
|
+
status: 401
|
|
603
|
+
});
|
|
604
|
+
}
|
|
605
|
+
console.error(error);
|
|
606
|
+
return Response.json({
|
|
607
|
+
error: 'Dataset operation failed'
|
|
608
|
+
}, {
|
|
609
|
+
status: 500
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
};
|
|
613
|
+
|
|
614
|
+
//# sourceMappingURL=datasets-route.js.map
|