@aglyn/plugins-data 1.0.0-beta.219 → 1.0.0-beta.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +10 -10
- package/src/lib/constants/bundle-common.d.ts +2 -0
- package/src/lib/constants/bundle-common.js +1 -0
- package/src/lib/constants/bundle-common.js.map +1 -1
- package/src/lib/declarations.console-server.d.ts +47 -0
- package/src/lib/declarations.console-server.js +74 -0
- package/src/lib/declarations.console-server.js.map +1 -0
- package/src/lib/declarations.server.d.ts +4 -0
- package/src/lib/declarations.server.js +11 -1
- package/src/lib/declarations.server.js.map +1 -1
- package/src/lib/model/dataset-csv.d.ts +36 -0
- package/src/lib/model/dataset-csv.js +85 -0
- package/src/lib/model/dataset-csv.js.map +1 -0
- package/src/lib/model/dataset-io.d.ts +8 -7
- package/src/lib/model/dataset-io.js +9 -8
- package/src/lib/model/dataset-io.js.map +1 -1
- package/src/lib/model/dataset-record-list.d.ts +40 -0
- package/src/lib/model/dataset-record-list.js +61 -0
- package/src/lib/model/dataset-record-list.js.map +1 -0
- package/src/lib/model/dataset-record.d.ts +33 -0
- package/src/lib/model/dataset-record.js +68 -0
- package/src/lib/model/dataset-record.js.map +1 -0
- package/src/lib/plugin.js +3 -0
- package/src/lib/plugin.js.map +1 -1
- package/src/lib/server/announce-dataset-records.d.ts +37 -0
- package/src/lib/server/announce-dataset-records.js +54 -0
- package/src/lib/server/announce-dataset-records.js.map +1 -0
- package/src/lib/server/api-v1/datasets.d.ts +18 -0
- package/src/lib/server/api-v1/datasets.js +646 -0
- package/src/lib/server/api-v1/datasets.js.map +1 -0
- package/src/lib/server/api-v1/openapi.d.ts +26 -0
- package/src/lib/server/api-v1/openapi.js +260 -0
- package/src/lib/server/api-v1/openapi.js.map +1 -0
- package/src/lib/server/api-v1/usage.d.ts +26 -0
- package/src/lib/server/api-v1/usage.js +33 -0
- package/src/lib/server/api-v1/usage.js.map +1 -0
- package/src/lib/server/custom-field-types.d.ts +39 -0
- package/src/lib/server/custom-field-types.js +41 -0
- package/src/lib/server/custom-field-types.js.map +1 -0
- package/src/lib/server/dataset-figures.d.ts +61 -0
- package/src/lib/server/dataset-figures.js +413 -0
- package/src/lib/server/dataset-figures.js.map +1 -0
- package/src/lib/server/dataset-record-index.d.ts +18 -0
- package/src/lib/server/dataset-record-index.js +62 -0
- package/src/lib/server/dataset-record-index.js.map +1 -0
- package/src/lib/server/dataset-storage-meter.d.ts +29 -0
- package/src/lib/server/dataset-storage-meter.js +82 -0
- package/src/lib/server/dataset-storage-meter.js.map +1 -0
- package/src/lib/server/datasets-export-route.d.ts +57 -0
- package/src/lib/server/datasets-export-route.js +229 -0
- package/src/lib/server/datasets-export-route.js.map +1 -0
- package/src/lib/server/datasets-route.d.ts +38 -0
- package/src/lib/server/datasets-route.js +614 -0
- package/src/lib/server/datasets-route.js.map +1 -0
- package/src/lib/server.d.ts +23 -0
- package/src/lib/server.js +54 -0
- package/src/lib/server.js.map +1 -0
- package/src/lib/site-bundle/datasets-site-bundle.server.js +2 -15
- package/src/lib/site-bundle/datasets-site-bundle.server.js.map +1 -1
- package/src/lib/usage-axes.d.ts +24 -0
- package/src/lib/usage-axes.js +32 -0
- package/src/lib/usage-axes.js.map +1 -0
|
@@ -0,0 +1,646 @@
|
|
|
1
|
+
import { _ as _extends } from "@swc/helpers/_/_extends";
|
|
2
|
+
/**
|
|
3
|
+
* @license
|
|
4
|
+
* Copyright 2026 Aglyn LLC
|
|
5
|
+
*
|
|
6
|
+
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
7
|
+
* you may not use this file except in compliance with the License.
|
|
8
|
+
* You may obtain a copy of the License at
|
|
9
|
+
*
|
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
|
11
|
+
*
|
|
12
|
+
* Unless required by applicable law or agreed to in writing, software
|
|
13
|
+
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
14
|
+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
15
|
+
* See the License for the specific language governing permissions and
|
|
16
|
+
* limitations under the License.
|
|
17
|
+
*/ import { checkDatasetQuota, checkEntitlement, checkQuota, coerceDocumentValues, createResourceUid, datasetIntegrityFields, datasetIntegrityUpdate, defaultScopeForNewResource, effectiveDatasetModel, newResourceScopeFields, validateDocument } from "@aglyn/aglyn/server";
|
|
18
|
+
import { ApiErrors, apiJson, dataStorageRefusal, listResponse } from "@aglyn/tenant-data-admin";
|
|
19
|
+
import { claimWrite, paginate, readJsonBody, requireScope, serialize } from "@aglyn/tenant-data-admin/server/api-v1-kit";
|
|
20
|
+
import { FieldValue, Timestamp } from "firebase-admin/firestore";
|
|
21
|
+
import { announceDatasetRecords as announceDatasetChange } from "../announce-dataset-records.js";
|
|
22
|
+
import { loadCustomFieldTypes } from "../custom-field-types.js";
|
|
23
|
+
/**
|
|
24
|
+
* The organization's datasets on the customer REST API, `/v1/datasets/…`
|
|
25
|
+
* (AGL-2126, AGL-3080): the datasets themselves and the records inside them.
|
|
26
|
+
* Registered from this plugin's console server declarations; the console's
|
|
27
|
+
* `/v1` router owns the pipeline in front — the key, the plan's API access,
|
|
28
|
+
* the request quota, the rate limit — and hands every request under
|
|
29
|
+
* `/v1/datasets` here.
|
|
30
|
+
*/ const datasetName = (data)=>{
|
|
31
|
+
var _ref, _data_displayName;
|
|
32
|
+
return (_ref = (_data_displayName = data.displayName) != null ? _data_displayName : data.name) != null ? _ref : '';
|
|
33
|
+
};
|
|
34
|
+
function datasetView(doc) {
|
|
35
|
+
var _doc_data, _data_fields, _serialize;
|
|
36
|
+
const data = (_doc_data = doc.data()) != null ? _doc_data : {};
|
|
37
|
+
return {
|
|
38
|
+
id: doc.id,
|
|
39
|
+
object: 'dataset',
|
|
40
|
+
name: datasetName(data),
|
|
41
|
+
fields: (_data_fields = data.fields) != null ? _data_fields : [],
|
|
42
|
+
created: (_serialize = serialize(data.createdAt)) != null ? _serialize : null
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
function recordView(doc) {
|
|
46
|
+
var _doc_data, _data_values, _serialize, _serialize1;
|
|
47
|
+
const data = (_doc_data = doc.data()) != null ? _doc_data : {};
|
|
48
|
+
return {
|
|
49
|
+
id: doc.id,
|
|
50
|
+
object: 'record',
|
|
51
|
+
values: serialize((_data_values = data.values) != null ? _data_values : {}),
|
|
52
|
+
created: (_serialize = serialize(data.createdAt)) != null ? _serialize : null,
|
|
53
|
+
updated: (_serialize1 = serialize(data.updatedAt)) != null ? _serialize1 : null
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function datasetsCollection(ctx) {
|
|
57
|
+
return ctx.firestore.collection('orgs').doc(ctx.orgId).collection('datasets');
|
|
58
|
+
}
|
|
59
|
+
/** Serialized `model` ceiling, matching the console's create route. */ const DATASET_MODEL_MAX_BYTES = 64 * 1024;
|
|
60
|
+
const DATASET_NAME_MAX = 120;
|
|
61
|
+
const DATASET_FIELDS_MAX = 100;
|
|
62
|
+
/**
|
|
63
|
+
* Validate the writable half of a dataset document (AGL-2126). Returns the
|
|
64
|
+
* cleaned values, or the per-field map `conventions.md` publishes under
|
|
65
|
+
* `validation_failed` — the same shape `validateDocument` produces for a
|
|
66
|
+
* record, so a client branches on one thing across the whole resource.
|
|
67
|
+
*
|
|
68
|
+
* `partial` is what separates PATCH from POST: a create must be told a name
|
|
69
|
+
* and at least one field, an update may send either alone. Absent keys are
|
|
70
|
+
* left alone rather than cleared, because a PATCH that silently emptied
|
|
71
|
+
* `fields` would take a dataset's schema away on a typo'd request body.
|
|
72
|
+
*/ function readDatasetInput(body, { partial }) {
|
|
73
|
+
const errors = {};
|
|
74
|
+
const values = {};
|
|
75
|
+
const hasName = body.name !== undefined;
|
|
76
|
+
if (hasName || !partial) {
|
|
77
|
+
var _body_name;
|
|
78
|
+
const name = String((_body_name = body.name) != null ? _body_name : '').trim().slice(0, DATASET_NAME_MAX);
|
|
79
|
+
if (!name) errors.name = 'Required';
|
|
80
|
+
else values.name = name;
|
|
81
|
+
}
|
|
82
|
+
const hasFields = body.fields !== undefined;
|
|
83
|
+
if (hasFields || !partial) {
|
|
84
|
+
if (!Array.isArray(body.fields)) {
|
|
85
|
+
errors.fields = 'Must be an array of field names';
|
|
86
|
+
} else {
|
|
87
|
+
const fields = body.fields.map((field)=>String(field).trim()).filter((field)=>field.length > 0).slice(0, DATASET_FIELDS_MAX);
|
|
88
|
+
if (fields.length === 0) errors.fields = 'At least one field is required';
|
|
89
|
+
else values.fields = fields;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (body.model !== undefined) {
|
|
93
|
+
if (!body.model || typeof body.model !== 'object') {
|
|
94
|
+
errors.model = 'Must be an object';
|
|
95
|
+
} else if (JSON.stringify(body.model).length > DATASET_MODEL_MAX_BYTES) {
|
|
96
|
+
errors.model = `Must serialize to under ${DATASET_MODEL_MAX_BYTES} bytes`;
|
|
97
|
+
} else {
|
|
98
|
+
values.model = body.model;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
return Object.keys(errors).length ? {
|
|
102
|
+
errors
|
|
103
|
+
} : {
|
|
104
|
+
values
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* `POST /v1/datasets` (AGL-2126) — the call that lets the API bootstrap itself.
|
|
109
|
+
*
|
|
110
|
+
* Until this shipped, `/v1` could create and edit RECORDS but not the dataset
|
|
111
|
+
* holding them, so an agency provisioning a client workspace, or a developer
|
|
112
|
+
* keeping a schema in source control, had to click a dataset into existence
|
|
113
|
+
* before their first API call could do anything. The console has been able to
|
|
114
|
+
* do this all along (`apps/console/app/api/orgs/datasets`, `create-dataset`).
|
|
115
|
+
*
|
|
116
|
+
* The gates are the console route's, not a looser set. `dataStore` is the
|
|
117
|
+
* entitlement, `checkDatasetQuota` is add-on aware, and — the one that is
|
|
118
|
+
* invisible until it bites — `newResourceScopeFields` stamps `visibleTo`.
|
|
119
|
+
* A dataset created without it matches no `array-contains-any` and therefore
|
|
120
|
+
* renders on NO site at all (AGL-1044): the API would happily create data
|
|
121
|
+
* that never appears anywhere, which is worse than refusing.
|
|
122
|
+
*/ async function createDataset(request, ctx) {
|
|
123
|
+
if (!checkEntitlement(ctx.org, 'dataStore')) {
|
|
124
|
+
return ApiErrors.planRequired({
|
|
125
|
+
message: 'Datasets are not included in this organization’s plan',
|
|
126
|
+
code: 'data_store',
|
|
127
|
+
headers: ctx.headers
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
const parsed = readDatasetInput(await readJsonBody(request), {
|
|
131
|
+
partial: false
|
|
132
|
+
});
|
|
133
|
+
if ('errors' in parsed) {
|
|
134
|
+
return ApiErrors.badRequest({
|
|
135
|
+
message: 'Dataset failed validation',
|
|
136
|
+
code: 'validation_failed',
|
|
137
|
+
fields: parsed.errors,
|
|
138
|
+
headers: ctx.headers
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
/*
|
|
142
|
+
* CLAIM ABOVE THE QUOTA (AGL-2296), and release on the refusal.
|
|
143
|
+
*
|
|
144
|
+
* This used to check the quota first and claim after, so that a refusal an
|
|
145
|
+
* integrator can act on never consumed their key — a plan refusal is the
|
|
146
|
+
* most retried failure there is, and it clears when somebody buys an
|
|
147
|
+
* add-on. That reasoning is right and is preserved by the `release()`
|
|
148
|
+
* below; the ORDERING it produced was not. A create that consumed the LAST
|
|
149
|
+
* included slot could not be retried at all: the retry re-counts, is now AT
|
|
150
|
+
* the band, and is refused before the claim is ever consulted — so the
|
|
151
|
+
* replay never happens and the integrator cannot tell whether the dataset
|
|
152
|
+
* exists. `conventions.md` publishes the opposite promise.
|
|
153
|
+
*
|
|
154
|
+
* Claiming first and releasing on each refusal gets both properties at
|
|
155
|
+
* once, which is the trade `deleteRecord` already argues for and
|
|
156
|
+
* `createContact` follows. Validation stays above: a deterministic 400 must
|
|
157
|
+
* never take a key at all.
|
|
158
|
+
*/ const collection = datasetsCollection(ctx);
|
|
159
|
+
const claimed = await claimWrite(ctx, '*', request.headers.get('Idempotency-Key'), 'datasets');
|
|
160
|
+
if ('replay' in claimed) return claimed.replay;
|
|
161
|
+
const { claim } = claimed;
|
|
162
|
+
try {
|
|
163
|
+
var _ctx_org;
|
|
164
|
+
const datasetCount = (await collection.count().get()).data().count;
|
|
165
|
+
const quota = checkDatasetQuota(ctx.org, datasetCount);
|
|
166
|
+
if (!quota.allowed) {
|
|
167
|
+
await claim.release();
|
|
168
|
+
return ApiErrors.planRequired({
|
|
169
|
+
message: `Dataset limit reached (${quota.limit}). ` + (quota.upgradeRequired ? 'Upgrade the plan to add more.' : `Buy extra datasets for $${quota.addonPriceUsd}/mo each, or upgrade.`),
|
|
170
|
+
code: 'dataset_quota',
|
|
171
|
+
headers: ctx.headers
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
const id = createResourceUid();
|
|
175
|
+
await collection.doc(id).create(_extends({
|
|
176
|
+
displayName: parsed.values.name,
|
|
177
|
+
fields: parsed.values.fields
|
|
178
|
+
}, parsed.values.model ? {
|
|
179
|
+
model: parsed.values.model
|
|
180
|
+
} : {}, newResourceScopeFields(defaultScopeForNewResource({
|
|
181
|
+
defaultResourceScope: (_ctx_org = ctx.org) == null ? void 0 : _ctx_org.defaultResourceScope,
|
|
182
|
+
hostId: null
|
|
183
|
+
})), {
|
|
184
|
+
createdAt: Timestamp.now()
|
|
185
|
+
}));
|
|
186
|
+
const view = datasetView(await collection.doc(id).get());
|
|
187
|
+
// Stored as 200 so a replay is distinguishable from the fresh 201 — the
|
|
188
|
+
// rule `conventions.md` publishes and `createRecord` already follows.
|
|
189
|
+
await claim.record(200, view);
|
|
190
|
+
return apiJson(view, {
|
|
191
|
+
status: 201,
|
|
192
|
+
headers: ctx.headers
|
|
193
|
+
});
|
|
194
|
+
} catch (error) {
|
|
195
|
+
// Same direction as every other v1 write: a stranded key is irreversible
|
|
196
|
+
// from outside, a duplicate dataset is one DELETE away.
|
|
197
|
+
await claim.release();
|
|
198
|
+
throw error;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* `PATCH /v1/datasets/{id}` — rename, re-field, or re-model. Takes no
|
|
203
|
+
* `Idempotency-Key` and does not need one, for `updateRecord`'s reason: the
|
|
204
|
+
* same body twice lands the same state AND returns the same `200`.
|
|
205
|
+
*/ async function updateDataset(request, ctx, datasetRef) {
|
|
206
|
+
const snap = await datasetRef.get();
|
|
207
|
+
if (!snap.exists) {
|
|
208
|
+
return ApiErrors.notFound({
|
|
209
|
+
message: 'No such dataset',
|
|
210
|
+
headers: ctx.headers
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
const parsed = readDatasetInput(await readJsonBody(request), {
|
|
214
|
+
partial: true
|
|
215
|
+
});
|
|
216
|
+
if ('errors' in parsed) {
|
|
217
|
+
return ApiErrors.badRequest({
|
|
218
|
+
message: 'Dataset failed validation',
|
|
219
|
+
code: 'validation_failed',
|
|
220
|
+
fields: parsed.errors,
|
|
221
|
+
headers: ctx.headers
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
const { name, fields, model } = parsed.values;
|
|
225
|
+
const update = {};
|
|
226
|
+
if (name !== undefined) update.displayName = name;
|
|
227
|
+
if (fields !== undefined) update.fields = fields;
|
|
228
|
+
if (model !== undefined) update.model = model;
|
|
229
|
+
// An empty body is a no-op, answered with the current dataset rather than a
|
|
230
|
+
// 400: a client re-sending an unchanged object should not have to special-
|
|
231
|
+
// case it, and there is no state to disagree about.
|
|
232
|
+
if (Object.keys(update).length > 0) await datasetRef.update(update);
|
|
233
|
+
// The fields and the model decide how every bound page formats and orders
|
|
234
|
+
// this dataset's rows, so a schema change is a change to those pages — and
|
|
235
|
+
// no publish follows a `/v1` write (AGL-3386). A rename alone renders
|
|
236
|
+
// nothing: the display name is the console's label for the dataset. Best
|
|
237
|
+
// effort, like the record writes' announce; the update is already stored.
|
|
238
|
+
if (fields !== undefined || model !== undefined) {
|
|
239
|
+
await announceDatasetChange({
|
|
240
|
+
firestore: ctx.firestore,
|
|
241
|
+
orgId: ctx.orgId,
|
|
242
|
+
datasetId: datasetRef.id
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
return apiJson(datasetView(await datasetRef.get()), {
|
|
246
|
+
headers: ctx.headers
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* `DELETE /v1/datasets/{id}` — refuses while records remain.
|
|
251
|
+
*
|
|
252
|
+
* A recursive delete is not something a single REST call should do quietly:
|
|
253
|
+
* the records are the customer's content, and one mistyped id would take all
|
|
254
|
+
* of them with no receipt naming what went. So this answers `409 conflict`
|
|
255
|
+
* with the count, and the integrator deletes the records first — with the
|
|
256
|
+
* same key semantics, through an endpoint that already exists.
|
|
257
|
+
*/ async function deleteDataset(request, ctx, datasetRef) {
|
|
258
|
+
const claimed = await claimWrite(ctx, datasetRef.id, request.headers.get('Idempotency-Key'), 'dataset-deletes');
|
|
259
|
+
if ('replay' in claimed) return claimed.replay;
|
|
260
|
+
const { claim } = claimed;
|
|
261
|
+
try {
|
|
262
|
+
const snap = await datasetRef.get();
|
|
263
|
+
if (!snap.exists) {
|
|
264
|
+
// Release: a wrong id is the integrator's to correct and retry with the
|
|
265
|
+
// same key, exactly as `deleteRecord` does.
|
|
266
|
+
await claim.release();
|
|
267
|
+
return ApiErrors.notFound({
|
|
268
|
+
message: 'No such dataset',
|
|
269
|
+
headers: ctx.headers
|
|
270
|
+
});
|
|
271
|
+
}
|
|
272
|
+
const records = (await datasetRef.collection('records').count().get()).data().count;
|
|
273
|
+
if (records > 0) {
|
|
274
|
+
// Release too — this refusal clears once the records are gone, and the
|
|
275
|
+
// retry that should then succeed must not replay the refusal.
|
|
276
|
+
await claim.release();
|
|
277
|
+
return ApiErrors.conflict({
|
|
278
|
+
message: `Dataset still holds ${records} record${records === 1 ? '' : 's'}. Delete them first.`,
|
|
279
|
+
code: 'dataset_not_empty',
|
|
280
|
+
headers: ctx.headers
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
await datasetRef.delete();
|
|
284
|
+
const view = {
|
|
285
|
+
id: datasetRef.id,
|
|
286
|
+
object: 'dataset',
|
|
287
|
+
deleted: true
|
|
288
|
+
};
|
|
289
|
+
await claim.record(200, view);
|
|
290
|
+
return apiJson(view, {
|
|
291
|
+
headers: ctx.headers
|
|
292
|
+
});
|
|
293
|
+
} catch (error) {
|
|
294
|
+
await claim.release();
|
|
295
|
+
throw error;
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
export async function handleDatasets(request, ctx, segments, url) {
|
|
299
|
+
const [, datasetId, sub, recordId] = segments;
|
|
300
|
+
// /v1/datasets
|
|
301
|
+
if (!datasetId) {
|
|
302
|
+
if (request.method === 'GET') {
|
|
303
|
+
const denied = requireScope(ctx, 'datasets:read');
|
|
304
|
+
if (denied) return denied;
|
|
305
|
+
const { docs, nextCursor } = await paginate(datasetsCollection(ctx), url);
|
|
306
|
+
return listResponse(docs.map(datasetView), nextCursor, ctx.headers);
|
|
307
|
+
}
|
|
308
|
+
if (request.method === 'POST') {
|
|
309
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
310
|
+
if (denied) return denied;
|
|
311
|
+
return createDataset(request, ctx);
|
|
312
|
+
}
|
|
313
|
+
return ApiErrors.methodNotAllowed({
|
|
314
|
+
headers: _extends({}, ctx.headers, {
|
|
315
|
+
Allow: 'GET, POST'
|
|
316
|
+
})
|
|
317
|
+
});
|
|
318
|
+
}
|
|
319
|
+
const datasetRef = datasetsCollection(ctx).doc(datasetId);
|
|
320
|
+
// /v1/datasets/{id}
|
|
321
|
+
if (!sub) {
|
|
322
|
+
if (request.method === 'GET') {
|
|
323
|
+
const denied = requireScope(ctx, 'datasets:read');
|
|
324
|
+
if (denied) return denied;
|
|
325
|
+
const snap = await datasetRef.get();
|
|
326
|
+
if (!snap.exists) {
|
|
327
|
+
return ApiErrors.notFound({
|
|
328
|
+
message: 'No such dataset',
|
|
329
|
+
headers: ctx.headers
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
return apiJson(datasetView(snap), {
|
|
333
|
+
headers: ctx.headers
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
if (request.method === 'PATCH') {
|
|
337
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
338
|
+
if (denied) return denied;
|
|
339
|
+
return updateDataset(request, ctx, datasetRef);
|
|
340
|
+
}
|
|
341
|
+
if (request.method === 'DELETE') {
|
|
342
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
343
|
+
if (denied) return denied;
|
|
344
|
+
return deleteDataset(request, ctx, datasetRef);
|
|
345
|
+
}
|
|
346
|
+
return ApiErrors.methodNotAllowed({
|
|
347
|
+
headers: _extends({}, ctx.headers, {
|
|
348
|
+
Allow: 'GET, PATCH, DELETE'
|
|
349
|
+
})
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
if (sub !== 'records') {
|
|
353
|
+
return ApiErrors.notFound({
|
|
354
|
+
message: `Unknown endpoint`,
|
|
355
|
+
headers: ctx.headers
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
const datasetSnap = await datasetRef.get();
|
|
359
|
+
if (!datasetSnap.exists) {
|
|
360
|
+
return ApiErrors.notFound({
|
|
361
|
+
message: 'No such dataset',
|
|
362
|
+
headers: ctx.headers
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
const recordsRef = datasetRef.collection('records');
|
|
366
|
+
// /v1/datasets/{id}/records
|
|
367
|
+
if (!recordId) {
|
|
368
|
+
if (request.method === 'GET') {
|
|
369
|
+
const denied = requireScope(ctx, 'datasets:read');
|
|
370
|
+
if (denied) return denied;
|
|
371
|
+
const { docs, nextCursor } = await paginate(recordsRef, url);
|
|
372
|
+
return listResponse(docs.map(recordView), nextCursor, ctx.headers);
|
|
373
|
+
}
|
|
374
|
+
if (request.method === 'POST') {
|
|
375
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
376
|
+
if (denied) return denied;
|
|
377
|
+
return createRecord(request, ctx, datasetSnap, recordsRef);
|
|
378
|
+
}
|
|
379
|
+
return ApiErrors.methodNotAllowed({
|
|
380
|
+
headers: ctx.headers
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
// /v1/datasets/{id}/records/{recordId}
|
|
384
|
+
const recordRef = recordsRef.doc(recordId);
|
|
385
|
+
if (request.method === 'GET') {
|
|
386
|
+
const denied = requireScope(ctx, 'datasets:read');
|
|
387
|
+
if (denied) return denied;
|
|
388
|
+
const snap = await recordRef.get();
|
|
389
|
+
if (!snap.exists) return ApiErrors.notFound({
|
|
390
|
+
message: 'No such record',
|
|
391
|
+
headers: ctx.headers
|
|
392
|
+
});
|
|
393
|
+
return apiJson(recordView(snap), {
|
|
394
|
+
headers: ctx.headers
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
if (request.method === 'PATCH') {
|
|
398
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
399
|
+
if (denied) return denied;
|
|
400
|
+
return updateRecord(request, ctx, datasetSnap, recordRef);
|
|
401
|
+
}
|
|
402
|
+
if (request.method === 'DELETE') {
|
|
403
|
+
const denied = requireScope(ctx, 'datasets:write');
|
|
404
|
+
if (denied) return denied;
|
|
405
|
+
return deleteRecord(request, ctx, datasetSnap.id, recordRef);
|
|
406
|
+
}
|
|
407
|
+
return ApiErrors.methodNotAllowed({
|
|
408
|
+
headers: ctx.headers
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
async function createRecord(request, ctx, datasetSnap, recordsRef) {
|
|
412
|
+
var _datasetSnap_data, _body_values;
|
|
413
|
+
const model = effectiveDatasetModel((_datasetSnap_data = datasetSnap.data()) != null ? _datasetSnap_data : {});
|
|
414
|
+
// A plugin's field validator only runs once its plugin has registered it.
|
|
415
|
+
await loadCustomFieldTypes(model, ctx.loadPluginSurfaces);
|
|
416
|
+
const body = await readJsonBody(request);
|
|
417
|
+
const coerced = coerceDocumentValues(model, (_body_values = body.values) != null ? _body_values : {});
|
|
418
|
+
const errors = validateDocument(model, coerced);
|
|
419
|
+
if (Object.keys(errors).length) {
|
|
420
|
+
return ApiErrors.badRequest({
|
|
421
|
+
message: 'Record failed validation',
|
|
422
|
+
headers: ctx.headers,
|
|
423
|
+
code: 'validation_failed',
|
|
424
|
+
// Name the offending fields (AGL-901): validateDocument already
|
|
425
|
+
// produces this map, and a bare 'something is wrong' on a 20-field
|
|
426
|
+
// record leaves an integrator bisecting their payload.
|
|
427
|
+
fields: errors
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
/*
|
|
431
|
+
* THE TWO QUOTAS THIS ROUTE DID NOT HAVE (AGL-2253).
|
|
432
|
+
*
|
|
433
|
+
* `POST /v1/datasets/{id}/records` counted rows only to compute `order` and
|
|
434
|
+
* checked nothing: not `recordsPerDataset`, not `dataStorageMbPerOrg`. The
|
|
435
|
+
* console route (`/api/orgs/datasets`) enforces both on the same write, and
|
|
436
|
+
* the tenant form path enforces the rows half — so `/v1` was the one door
|
|
437
|
+
* into `orgs/{id}/datasets/{id}/records` with no cap on it, and a customer
|
|
438
|
+
* could blow a limit through the REST API that the UI refuses.
|
|
439
|
+
*
|
|
440
|
+
* They sit BELOW the idempotency claim (AGL-2296), which is a change from
|
|
441
|
+
* how they shipped. Checking them first protected the key on a refusal —
|
|
442
|
+
* right, and kept by the `release()` calls below — but it made a create
|
|
443
|
+
* that consumed the LAST included row impossible to retry: the retry
|
|
444
|
+
* re-counts, is now AT the band, and is refused before the claim is
|
|
445
|
+
* consulted, so the replay never happens. The bytes leg is worse in
|
|
446
|
+
* practice, because a bulk import crosses the storage band mid-run and
|
|
447
|
+
* every retry from that point on is refused rather than replayed.
|
|
448
|
+
*
|
|
449
|
+
* The row count is reused for `order`, so this adds no read on the rows
|
|
450
|
+
* leg. The bytes leg adds no read either on any plan that meters the
|
|
451
|
+
* overage; see `dataStorageRefusal`.
|
|
452
|
+
*/ // Idempotency: replay a prior create for the same key instead of
|
|
453
|
+
// duplicating. Claimed HERE, below validation, so a deterministic 400 never
|
|
454
|
+
// takes the key at all — an integrator fixes the payload and retries with
|
|
455
|
+
// the same key, exactly as the POS cashier does (AGL-1691).
|
|
456
|
+
const claimed = await claimWrite(ctx, datasetSnap.id, request.headers.get('Idempotency-Key'), 'records');
|
|
457
|
+
if ('replay' in claimed) return claimed.replay;
|
|
458
|
+
const { claim } = claimed;
|
|
459
|
+
try {
|
|
460
|
+
const order = (await recordsRef.count().get()).data().count;
|
|
461
|
+
const recordQuota = checkQuota(ctx.org, 'recordsPerDataset', order);
|
|
462
|
+
if (!recordQuota.allowed) {
|
|
463
|
+
await claim.release();
|
|
464
|
+
return ApiErrors.planRequired({
|
|
465
|
+
message: `Record limit reached (${recordQuota.limit}). Upgrade the plan to add more.`,
|
|
466
|
+
code: 'record_quota',
|
|
467
|
+
headers: ctx.headers
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
const storageRefusal = await dataStorageRefusal(ctx.org, ctx.firestore.collection('orgs').doc(ctx.orgId));
|
|
471
|
+
if (storageRefusal) {
|
|
472
|
+
await claim.release();
|
|
473
|
+
return ApiErrors.planRequired({
|
|
474
|
+
message: storageRefusal.basis === 'always' ? 'Dataset storage is not included in this organization’s plan' : `Dataset storage limit reached (${storageRefusal.includedMb} MB). Upgrade the plan to add more.`,
|
|
475
|
+
code: 'data_storage_quota',
|
|
476
|
+
headers: ctx.headers
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
const recordId = createResourceUid();
|
|
480
|
+
await recordsRef.doc(recordId).create(_extends({
|
|
481
|
+
values: coerced
|
|
482
|
+
}, datasetIntegrityFields(model, coerced), {
|
|
483
|
+
order,
|
|
484
|
+
createdAt: Timestamp.now(),
|
|
485
|
+
updatedAt: Timestamp.now()
|
|
486
|
+
}));
|
|
487
|
+
const created = await recordsRef.doc(recordId).get();
|
|
488
|
+
const view = recordView(created);
|
|
489
|
+
// AGL-2462 recorded that a `/v1` write cannot publish, and the ONLY thing
|
|
490
|
+
// that made it visible was the hour. It announces now (AGL-3113): the
|
|
491
|
+
// pages repeating over this dataset are dropped, on every site it is
|
|
492
|
+
// shared with. Best effort — the record is already stored.
|
|
493
|
+
await announceDatasetChange({
|
|
494
|
+
firestore: ctx.firestore,
|
|
495
|
+
orgId: ctx.orgId,
|
|
496
|
+
datasetId: datasetSnap.id
|
|
497
|
+
});
|
|
498
|
+
// Recorded as 200, never the 201 this answers: `conventions.md` publishes
|
|
499
|
+
// the status as how a client tells a fresh create from a replay, and that
|
|
500
|
+
// is the contract integrations branch on.
|
|
501
|
+
await claim.record(200, view);
|
|
502
|
+
return apiJson(view, {
|
|
503
|
+
status: 201,
|
|
504
|
+
headers: ctx.headers
|
|
505
|
+
});
|
|
506
|
+
} catch (error) {
|
|
507
|
+
/*
|
|
508
|
+
* Release on EVERY failure, including one whose outcome we cannot know —
|
|
509
|
+
* the deliberate opposite of the refund (AGL-1696, `2dd52f01c`), which
|
|
510
|
+
* strands the key on a throw. Same reasoning, applied where its money term
|
|
511
|
+
* is absent.
|
|
512
|
+
*
|
|
513
|
+
* NOTHING on `/v1` moves money — the write surface is datasets, records,
|
|
514
|
+
* the form-submission `read` flag and contacts (AGL-2276), while orders
|
|
515
|
+
* and products stay read-only precisely because writing them would — so
|
|
516
|
+
* "a released key costs a second refund" has no analogue here. The costs invert instead. A duplicate record is visible and
|
|
517
|
+
* reversible BY THE INTEGRATOR with `DELETE /v1/datasets/{id}/records/{id}`
|
|
518
|
+
* — same API, same scope, one extra call. A stranded key is irreversible
|
|
519
|
+
* from outside: keys are documented as never expiring, and an integrator
|
|
520
|
+
* derives them from their own upstream event ids precisely so retries
|
|
521
|
+
* dedupe, so a stranded key means that event can never be written at all.
|
|
522
|
+
*
|
|
523
|
+
* Residual: a process killed between the claim and the record strands the
|
|
524
|
+
* key regardless. `createdAtMs` is written so a sweeper can reap those.
|
|
525
|
+
*/ await claim.release();
|
|
526
|
+
throw error;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
async function updateRecord(request, ctx, datasetSnap, recordRef) {
|
|
530
|
+
var _datasetSnap_data, _snap_get, _body_values;
|
|
531
|
+
const snap = await recordRef.get();
|
|
532
|
+
if (!snap.exists) return ApiErrors.notFound({
|
|
533
|
+
message: 'No such record',
|
|
534
|
+
headers: ctx.headers
|
|
535
|
+
});
|
|
536
|
+
const model = effectiveDatasetModel((_datasetSnap_data = datasetSnap.data()) != null ? _datasetSnap_data : {});
|
|
537
|
+
// A plugin's field validator only runs once its plugin has registered it.
|
|
538
|
+
await loadCustomFieldTypes(model, ctx.loadPluginSurfaces);
|
|
539
|
+
const body = await readJsonBody(request);
|
|
540
|
+
// PATCH merges the supplied fields over the stored values.
|
|
541
|
+
const merged = _extends({}, (_snap_get = snap.get('values')) != null ? _snap_get : {}, coerceDocumentValues(model, (_body_values = body.values) != null ? _body_values : {}));
|
|
542
|
+
const errors = validateDocument(model, merged);
|
|
543
|
+
if (Object.keys(errors).length) {
|
|
544
|
+
return ApiErrors.badRequest({
|
|
545
|
+
message: 'Record failed validation',
|
|
546
|
+
headers: ctx.headers,
|
|
547
|
+
code: 'validation_failed',
|
|
548
|
+
// Name the offending fields (AGL-901): validateDocument already
|
|
549
|
+
// produces this map, and a bare 'something is wrong' on a 20-field
|
|
550
|
+
// record leaves an integrator bisecting their payload.
|
|
551
|
+
fields: errors
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
await recordRef.update(_extends({
|
|
555
|
+
values: merged
|
|
556
|
+
}, datasetIntegrityUpdate(model, merged, FieldValue.delete()), {
|
|
557
|
+
updatedAt: Timestamp.now()
|
|
558
|
+
}));
|
|
559
|
+
const updated = await recordRef.get();
|
|
560
|
+
// An edited row is as stale on a live page as a new one (AGL-3113).
|
|
561
|
+
await announceDatasetChange({
|
|
562
|
+
firestore: ctx.firestore,
|
|
563
|
+
orgId: ctx.orgId,
|
|
564
|
+
datasetId: datasetSnap.id
|
|
565
|
+
});
|
|
566
|
+
return apiJson(recordView(updated), {
|
|
567
|
+
headers: ctx.headers
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
/**
|
|
571
|
+
* Delete a record, and let a retry of the SAME attempt say so (AGL-1710).
|
|
572
|
+
*
|
|
573
|
+
* This used to read, 404 if absent, then delete. The state after two calls was
|
|
574
|
+
* right and the response was not: an integrator whose first response was lost
|
|
575
|
+
* to a timeout retried, got `404 No such record`, and had no way to separate
|
|
576
|
+
* "already deleted, you're fine" from "that id was wrong and nothing was ever
|
|
577
|
+
* deleted". Both readings prescribe different actions and the wrong one —
|
|
578
|
+
* escalate, or re-sync the whole dataset — is the one people pick.
|
|
579
|
+
*
|
|
580
|
+
* The obvious fix, `204` (or a `deleted: true` 200) for ANY missing record, is
|
|
581
|
+
* rejected on two counts. It is a BREAKING change: `datasets.md` publishes
|
|
582
|
+
* `404 not_found "No such record"` on this path and clients branch on it. And
|
|
583
|
+
* it spends a signal to buy one — a caller that never saw the resource would
|
|
584
|
+
* get a success for a typo'd id, losing the only feedback the API gives that
|
|
585
|
+
* it is asking about the wrong thing. Consistency points the same way: `GET`
|
|
586
|
+
* and `PATCH` on this very path answer 404 for a missing record, and every
|
|
587
|
+
* sibling `DELETE` on `/v1` — the dataset, the form submission, the contact —
|
|
588
|
+
* answers the same way, so there is no convention a 204 would be matching.
|
|
589
|
+
*
|
|
590
|
+
* So the key identifies the ATTEMPT rather than the resource, which is what
|
|
591
|
+
* the shared claim is already for. A retry of the attempt that did the
|
|
592
|
+
* deleting replays its receipt; a wrong id still 404s. That expands the
|
|
593
|
+
* published contract instead of changing it — a caller sending no header sees
|
|
594
|
+
* byte-identical behaviour.
|
|
595
|
+
*
|
|
596
|
+
* The claim is taken ABOVE the existence check, which is the one place this
|
|
597
|
+
* deliberately diverges from `createRecord` (which claims below validation, so
|
|
598
|
+
* a deterministic 400 never burns a key). It has to: the record a retry asks
|
|
599
|
+
* about is precisely the one the first attempt removed, so a claim consulted
|
|
600
|
+
* after the existence check would 404 the retry without ever reaching the
|
|
601
|
+
* replay — the bug verbatim. The cost is taking-and-releasing on a genuine
|
|
602
|
+
* miss, which is the trade the ordering requires.
|
|
603
|
+
*/ async function deleteRecord(request, ctx, datasetId, recordRef) {
|
|
604
|
+
const claimed = await claimWrite(ctx, datasetId, request.headers.get('Idempotency-Key'), 'record-deletes');
|
|
605
|
+
if ('replay' in claimed) return claimed.replay;
|
|
606
|
+
const { claim } = claimed;
|
|
607
|
+
try {
|
|
608
|
+
const snap = await recordRef.get();
|
|
609
|
+
if (!snap.exists) {
|
|
610
|
+
// Release, so the miss does not consume the key: the integrator corrects
|
|
611
|
+
// the id and retries with the same one, exactly as a create's 400 lets
|
|
612
|
+
// them correct the payload. A burned key here would be worse than the
|
|
613
|
+
// 404 — keys never expire, and integrators derive them from upstream
|
|
614
|
+
// event ids, so the corrected delete could never be sent at all.
|
|
615
|
+
await claim.release();
|
|
616
|
+
return ApiErrors.notFound({
|
|
617
|
+
message: 'No such record',
|
|
618
|
+
headers: ctx.headers
|
|
619
|
+
});
|
|
620
|
+
}
|
|
621
|
+
await recordRef.delete();
|
|
622
|
+
// A row that is gone is the same staleness as a row that changed: the
|
|
623
|
+
// page goes on rendering it until its cache is dropped (AGL-3113).
|
|
624
|
+
await announceDatasetChange({
|
|
625
|
+
firestore: ctx.firestore,
|
|
626
|
+
orgId: ctx.orgId,
|
|
627
|
+
datasetId
|
|
628
|
+
});
|
|
629
|
+
const view = {
|
|
630
|
+
id: recordRef.id,
|
|
631
|
+
object: 'record',
|
|
632
|
+
deleted: true
|
|
633
|
+
};
|
|
634
|
+
await claim.record(200, view);
|
|
635
|
+
return apiJson(view, {
|
|
636
|
+
headers: ctx.headers
|
|
637
|
+
});
|
|
638
|
+
} catch (error) {
|
|
639
|
+
// Same direction as `createRecord`: a failed attempt gives the key back.
|
|
640
|
+
// Nothing here moves money, and a stranded key is the irreversible failure.
|
|
641
|
+
await claim.release();
|
|
642
|
+
throw error;
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
//# sourceMappingURL=datasets.js.map
|