@aglyn/plugins-data 1.0.0-beta.220 → 1.0.0-beta.222

package/src/lib/transfer/dataset-transfer.server.js

@@ -0,0 +1,930 @@
+import { _ as _extends } from "@swc/helpers/_/_extends";
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ import { createHash } from "node:crypto";
+import { TRANSFER_ID_FIELD, buildTransferPlan, matchLookupKey, planTransferUndo, rankTransferLookupSuggestions, transferLookupKey, transferLookupNewName, transferResourceInstanceOf } from "@aglyn/aglyn/data-transfer";
+import { checkQuota, ensureDeclaredCustomFieldTypes, memberCanSee } from "@aglyn/aglyn/server";
+import { dataStorageRefusal, firebaseAdmin, isServerReleaseFlagOnForOrg, resolveOrgMembership } from "@aglyn/tenant-data-admin";
+import { applyListQuery } from "@aglyn/tenant-data-admin/server/list-query";
+import { TransferEngineError } from "@aglyn/tenant-data-admin/server/transfer-jobs";
+import { FieldPath, FieldValue, Timestamp } from "firebase-admin/firestore";
+import { datasetRecordFilter, planRecordQuery } from "../components/dataset-record-filter.js";
+import { datasetFilterToken, datasetFilterValuePath, datasetFilterWords, datasetIntegrityFields, datasetIntegrityUpdate, effectiveDatasetModel } from "../model/dataset-models.js";
+import { datasetDisplayName, describeDatasetRecordErrors } from "../model/datasets.js";
+import { fillRecordAddresses } from "../record-pages/record-pages.js";
+import { announceDatasetRecords } from "../server/announce-dataset-records.js";
+import { datasetTransferResourceKey } from "./dataset-transfer-key.js";
+import { datasetFieldIdOf, datasetFieldOfPicklist, datasetHoldsKey, datasetMatchKeyOffer, datasetMatchKeyQuery, datasetOptionsPicklist, datasetRecordTransferValues, datasetStorageValues, datasetTransferCatalog, datasetTransferFieldId, datasetWriteErrors, isOptionsField, limitDatasetCreates, refuseInvalidDatasetRows } from "./dataset-transfer-model.js";
+const productionDeps = ()=>({
+        firestore: firebaseAdmin.app().firestore()
+    });
+/** The page an export reads when the engine names none. */ const PAGE_ROWS = 500;
+/** Firestore's cap on the values of one `in` query. */ const IN_MAX = 30;
+/** The most documents one `getAll` reads. */ const GET_ALL_MAX = 300;
+/** What a merging record write names, so each field is replaced whole. */ const RECORD_MERGE_FIELDS = [
+    'values',
+    'referencedIds',
+    'filterKeys',
+    'filterValues',
+    'updatedAt'
+];
+const chunked = (items, size)=>{
+    const out = [];
+    for(let at = 0; at < items.length; at += size)out.push(items.slice(at, at + size));
+    return out;
+};
+const notFound = ()=>new TransferEngineError('notFound', 404, 'No such dataset.');
+/** The nanoid alphabet `createResourceUid` names documents from. */ const ID_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_-';
+/**
+ * The id a record created by an import's row is given: ten characters of the
+ * resource-id alphabet, derived from the job and the row, so a retried chunk
+ * names the record it already wrote rather than writing a second one.
+ */ export function transferRecordId(jobId, row) {
+    const digest = createHash('sha256').update(`${jobId}:${row}`).digest();
+    let id = '';
+    for(let at = 0; at < 10; at += 1)id += ID_ALPHABET[digest[at] % ID_ALPHABET.length];
+    return id;
+}
+/**
+ * The dataset per context object: the engine hands one context to every
+ * hook a step runs, so an export's pages and a plan's lookups read the
+ * dataset and the member once.
+ */ const loaded = new WeakMap();
+async function readDataset(deps, ctx) {
+    const datasetId = transferResourceInstanceOf(ctx);
+    if (!datasetId) throw notFound();
+    const orgRef = deps.firestore.collection('orgs').doc(ctx.orgId);
+    const ref = orgRef.collection('datasets').doc(datasetId);
+    const snapshot = await ref.get();
+    const data = snapshot.exists ? snapshot.data() : null;
+    if (!data || data['deletedAt'] != null) throw notFound();
+    const visibleTo = Array.isArray(data['visibleTo']) ? data['visibleTo'] : [];
+    /*
+   * The member has to SEE the dataset, as on every other door onto its
+   * records (`datasets-route.ts`): the gate asked about the workspace, and a
+   * collaborator scoped to one site passes it. The Admin SDK reads past the
+   * rules, so this IS the enforcement, answered with the same 404 as a
+   * dataset that does not exist. No membership is staff (the gate admitted
+   * them) or the sweep resuming a job; the data store's release flag holds
+   * for everyone else.
+   */ if (ctx.actorUid) {
+        var _this;
+        const member = (_this = await resolveOrgMembership(ctx.actorUid, ctx.orgId)) == null ? void 0 : _this.member;
+        if (member) {
+            if (!memberCanSee(member, visibleTo)) throw notFound();
+            if (!await isServerReleaseFlagOnForOrg('release_data_store', ctx.orgId)) {
+                throw new TransferEngineError('notFound', 404, 'Datasets are not available on this workspace.');
+            }
+        }
+    }
+    return {
+        id: datasetId,
+        label: datasetDisplayName(data) || 'Fields',
+        model: effectiveDatasetModel(data),
+        visibleTo,
+        orgRef,
+        ref,
+        records: ref.collection('records')
+    };
+}
+function datasetFor(deps, ctx) {
+    let pending = loaded.get(ctx);
+    if (!pending) {
+        pending = readDataset(deps, ctx);
+        loaded.set(ctx, pending);
+        pending.catch(()=>loaded.delete(ctx));
+    }
+    return pending;
+}
+/** The context a referenced dataset is asked in: the same job, actor and site, as that dataset. */ function targetContext(ctx, datasetId) {
+    return _extends({}, ctx, {
+        resource: datasetTransferResourceKey(datasetId)
+    });
+}
+/**
+ * Each dataset the model references, as the person may use it: whether they
+ * can read it (the same check as opening it), and its model when they can.
+ */ async function datasetReferenceTargets(deps, ctx, model) {
+    var _model_fields;
+    const out = {};
+    for (const field of Object.values((_model_fields = model.fields) != null ? _model_fields : {})){
+        var _field_reference;
+        const targetId = (field == null ? void 0 : field.type) === 'reference' ? (_field_reference = field.reference) == null ? void 0 : _field_reference.datasetId : undefined;
+        if (!targetId || out[targetId]) continue;
+        try {
+            out[targetId] = {
+                readable: true,
+                model: (await datasetFor(deps, targetContext(ctx, targetId))).model
+            };
+        } catch (error) {
+            if (!(error instanceof TransferEngineError && error.code === 'notFound')) throw error;
+            out[targetId] = {
+                readable: false
+            };
+        }
+    }
+    return out;
+}
+/** A record document as transfer values. */ function viewOf(model, doc) {
+    var _doc_data, _data_values;
+    const data = (_doc_data = doc.data()) != null ? _doc_data : {};
+    return datasetRecordTransferValues(model, {
+        id: doc.id,
+        values: (_data_values = data['values']) != null ? _data_values : {},
+        createdAt: data['createdAt'],
+        updatedAt: data['updatedAt']
+    });
+}
+async function getAll(firestore, refs) {
+    const out = [];
+    for (const slice of chunked(refs, GET_ALL_MAX))if (slice.length) out.push(...await firestore.getAll(...slice));
+    return out;
+}
+function readFilter(filter) {
+    const clauses = Array.isArray(filter['clauses']) ? filter['clauses'] : [];
+    const search = Array.isArray(filter['search']) ? filter['search'] : [];
+    const clause = (entry)=>{
+        const one = entry;
+        return Boolean(one) && typeof (one == null ? void 0 : one['field']) === 'string' && typeof (one == null ? void 0 : one['op']) === 'string' && typeof (one == null ? void 0 : one['value']) === 'string';
+    };
+    if (!clauses.every(clause) || !search.every((word)=>typeof word === 'string')) {
+        throw new TransferEngineError('invalid', 400, 'The filter could not be read.');
+    }
+    return {
+        clauses: clauses,
+        search: search
+    };
+}
+/**
+ * Every record, or the ones the records table's filter matches — the same
+ * one query the table pages (`planRecordQuery`), ordered by document id.
+ * What the table could not put on its query it did not apply either, so
+ * neither does the export.
+ */ function recordsQuery(dataset, options) {
+    if (!(options == null ? void 0 : options.filter)) return dataset.records.orderBy(FieldPath.documentId());
+    const filter = readFilter(options.filter);
+    const { plan } = planRecordQuery(dataset.model, datasetRecordFilter(dataset.model), filter.clauses, filter.search);
+    return applyListQuery(dataset.records, plan);
+}
+/** A collaborator's scope tokens must reach the dataset; records carry no scope of their own. */ function assertScope(dataset, options) {
+    if ((options == null ? void 0 : options.scopeTokens) && !options.scopeTokens.some((token)=>dataset.visibleTo.includes(token))) throw notFound();
+}
+function pick(view, fieldIds) {
+    return Object.fromEntries(fieldIds.filter((fieldId)=>view[fieldId] !== undefined).map((fieldId)=>[
+            fieldId,
+            view[fieldId]
+        ]));
+}
+export async function countDatasetRecords(deps, ctx, options) {
+    var _data_count;
+    const dataset = await datasetFor(deps, ctx);
+    assertScope(dataset, options);
+    if (options.ids) {
+        let total = 0;
+        for (const slice of chunked(options.ids, IN_MAX)){
+            var _data_count1;
+            total += Number((_data_count1 = (await dataset.records.where(FieldPath.documentId(), 'in', slice).count().get()).data().count) != null ? _data_count1 : 0);
+        }
+        return total;
+    }
+    return Number((_data_count = (await recordsQuery(dataset, options).count().get()).data().count) != null ? _data_count : 0);
+}
+export async function readDatasetPage(deps, ctx, cursor, fieldIds, options) {
+    var _ref;
+    const dataset = await datasetFor(deps, ctx);
+    assertScope(dataset, options);
+    const size = Math.max(1, Math.min((_ref = options == null ? void 0 : options.pageSize) != null ? _ref : PAGE_ROWS, PAGE_ROWS));
+    if (options == null ? void 0 : options.ids) {
+        // The selection, in the order it was made; the cursor is how far through it.
+        const offset = Math.max(0, Number(cursor != null ? cursor : 0) || 0);
+        const slice = options.ids.slice(offset, offset + size);
+        const docs = await getAll(deps.firestore, slice.map((id)=>dataset.records.doc(id)));
+        return {
+            rows: docs.filter((doc)=>doc.exists).map((doc)=>pick(viewOf(dataset.model, doc), fieldIds)),
+            next: offset + size < options.ids.length ? String(offset + size) : null
+        };
+    }
+    let query = recordsQuery(dataset, options);
+    if (cursor) query = query.startAfter(cursor);
+    const snapshot = await query.limit(size).get();
+    const docs = snapshot.docs;
+    return {
+        rows: docs.map((doc)=>pick(viewOf(dataset.model, doc), fieldIds)),
+        next: docs.length === size ? docs[docs.length - 1].id : null
+    };
+}
+/*------------------------------------------
+ * Matching
+ *-----------------------------------------*/ export async function lookupDatasetRecords(deps, ctx, requests) {
+    const dataset = await datasetFor(deps, ctx);
+    const lookup = new Map();
+    const records = new Map();
+    const file = (fieldId, normalized, id)=>{
+        var _lookup_get;
+        const at = matchLookupKey(fieldId, normalized);
+        const ids = (_lookup_get = lookup.get(at)) != null ? _lookup_get : [];
+        if (!ids.includes(id)) ids.push(id);
+        lookup.set(at, ids);
+    };
+    for (const request of requests){
+        var _datasetFieldIdOf;
+        if (!request.values.length) continue;
+        if (request.fieldId === TRANSFER_ID_FIELD) {
+            const docs = await getAll(deps.firestore, request.values.map((id)=>dataset.records.doc(id)));
+            for (const doc of docs){
+                if (!doc.exists) continue;
+                records.set(doc.id, viewOf(dataset.model, doc));
+                file(request.fieldId, doc.id, doc.id);
+            }
+            continue;
+        }
+        // Asked of `filterValues`, the index the records table's own filters
+        // read; a value clipped there is held to the whole value below.
+        const asked = new Map();
+        for (const normalized of request.values){
+            var _asked_get;
+            const query = datasetMatchKeyQuery(dataset.model, request, normalized);
+            if (!query) continue;
+            asked.set(query.value, [
+                ...(_asked_get = asked.get(query.value)) != null ? _asked_get : [],
+                normalized
+            ]);
+        }
+        const path = datasetFilterValuePath((_datasetFieldIdOf = datasetFieldIdOf(request.fieldId)) != null ? _datasetFieldIdOf : '');
+        if (!path) continue;
+        for (const slice of chunked([
+            ...asked.keys()
+        ], IN_MAX)){
+            const snapshot = await dataset.records.where(path, 'in', slice).get();
+            for (const doc of snapshot.docs){
+                const view = viewOf(dataset.model, doc);
+                let held = false;
+                for (const normalized of slice.flatMap((value)=>{
+                    var _asked_get;
+                    return (_asked_get = asked.get(value)) != null ? _asked_get : [];
+                })){
+                    if (!datasetHoldsKey(view, request, normalized)) continue;
+                    file(request.fieldId, normalized, doc.id);
+                    held = true;
+                }
+                if (held) records.set(doc.id, view);
+            }
+        }
+    }
+    return {
+        lookup,
+        records
+    };
+}
+/** The most distinct word queries one suggestion request runs. */ const SUGGEST_QUERIES_MAX = 200;
+/** The records one word query reads. */ const SUGGEST_READ = 20;
+/** The words of a value its candidates are asked by. */ const SUGGEST_WORDS = 2;
+/** The leading characters a word is also asked by, so a later misspelling still finds it. */ const SUGGEST_STEM = 3;
+/**
+ * Records named LIKE each value a reference could not find, for the person
+ * to pick from: each plain text `by` field asked for records with a word that
+ * starts with one of the value's first words, or with its first few letters
+ * (the `filterKeys` prefix tokens the records table's `contains` reads), then
+ * ranked by how close the whole name is. Values past the query budget get none.
+ */ export async function suggestDatasetRecords(deps, ctx, request) {
+    const dataset = await datasetFor(deps, ctx);
+    const fields = request.by.map((transferId)=>datasetFieldIdOf(transferId)).filter((fieldId)=>{
+        var _dataset_model_fields;
+        const field = fieldId ? (_dataset_model_fields = dataset.model.fields) == null ? void 0 : _dataset_model_fields[fieldId] : undefined;
+        return (field == null ? void 0 : field.type) === 'text' && !isOptionsField(field);
+    });
+    const asked = new Map();
+    const ask = (fieldId, token)=>{
+        const at = `${fieldId}\u0000${token}`;
+        let pending = asked.get(at);
+        if (!pending) {
+            pending = dataset.records.where('filterKeys', 'array-contains', token).limit(SUGGEST_READ).get().then((snapshot)=>snapshot.docs.map((doc)=>{
+                    var _fieldId, _doc_get;
+                    return {
+                        recordId: doc.id,
+                        label: String((_fieldId = ((_doc_get = doc.get('values')) != null ? _doc_get : {})[fieldId]) != null ? _fieldId : '').trim()
+                    };
+                }));
+            asked.set(at, pending);
+        }
+        return pending;
+    };
+    const answer = {};
+    for (const value of request.values){
+        const candidates = [];
+        for (const fieldId of fields){
+            for (const word of datasetFilterWords(value).slice(0, SUGGEST_WORDS)){
+                const stem = Array.from(word).slice(0, SUGGEST_STEM).join('');
+                for (const asWord of new Set([
+                    word,
+                    stem
+                ])){
+                    const token = datasetFilterToken(dataset.model, {
+                        field: fieldId,
+                        op: 'contains',
+                        value: asWord
+                    });
+                    if (!token || asked.size >= SUGGEST_QUERIES_MAX && !asked.has(`${fieldId}\u0000${token}`)) continue;
+                    candidates.push(...(await ask(fieldId, token)).filter((candidate)=>candidate.label));
+                }
+            }
+        }
+        answer[value] = rankTransferLookupSuggestions(value, candidates, {
+            limit: 5
+        });
+    }
+    return answer;
+}
+/*------------------------------------------
+ * Options fields as picklists
+ *-----------------------------------------*/ export async function datasetPicklists(deps, ctx, picklistIds) {
+    const { model } = await datasetFor(deps, ctx);
+    const out = {};
+    for (const picklistId of picklistIds){
+        var _datasetFieldOfPicklist;
+        var _model_fields;
+        const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[(_datasetFieldOfPicklist = datasetFieldOfPicklist(picklistId)) != null ? _datasetFieldOfPicklist : ''];
+        if (field) out[picklistId] = datasetOptionsPicklist(field);
+    }
+    return out;
+}
+/**
+ * The values the person chose to add to an options field, added to the
+ * field's options in the dataset's model — the schema the Schema dialog
+ * edits — before the import's first write. A value already there is left.
+ */ export async function addDatasetOptions(deps, ctx, picklistId, values) {
+    const dataset = await datasetFor(deps, ctx);
+    const fieldId = datasetFieldOfPicklist(picklistId);
+    if (!fieldId) return;
+    await deps.firestore.runTransaction(async (tx)=>{
+        var _snapshot_data, _ref;
+        var _model_fields, _field_validation;
+        const snapshot = await tx.get(dataset.ref);
+        const model = effectiveDatasetModel((_snapshot_data = snapshot.data()) != null ? _snapshot_data : {});
+        const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[fieldId];
+        if (!field) return;
+        const options = [
+            ...(_ref = (_field_validation = field.validation) == null ? void 0 : _field_validation.options) != null ? _ref : []
+        ];
+        const added = values.map((value)=>value.label).filter((label)=>label && !options.includes(label));
+        if (!added.length) return;
+        tx.update(dataset.ref, {
+            model: _extends({}, model, {
+                fields: _extends({}, model.fields, {
+                    [fieldId]: _extends({}, field, {
+                        validation: _extends({}, field.validation, {
+                            options: [
+                                ...options,
+                                ...added
+                            ]
+                        })
+                    })
+                })
+            }),
+            updatedAt: Timestamp.now()
+        });
+    });
+    // The writes that follow validate against the options just added.
+    loaded.delete(ctx);
+}
+/*------------------------------------------
+ * The dry run
+ *-----------------------------------------*/ /**
+ * The records this import may create, from the plan's `recordsPerDataset`
+ * against the dataset's real size — and none while the data storage band is
+ * full — so the dry run fails what Apply would refuse.
+ */ async function createLimits(dataset) {
+    var _data_count;
+    const org = (await dataset.orgRef.get()).data();
+    if (await dataStorageRefusal(org, dataset.orgRef)) return {
+        maxCreates: 0
+    };
+    const live = Number((_data_count = (await dataset.records.count().get()).data().count) != null ? _data_count : 0);
+    const { remaining } = checkQuota(org, 'recordsPerDataset', live);
+    return Number.isFinite(remaining) ? {
+        maxCreates: remaining
+    } : {};
+}
+const failed = (row, reason, message)=>({
+        result: _extends({
+            row: row.index,
+            outcome: 'failed'
+        }, row.recordId ? {
+            recordId: row.recordId
+        } : {}, {
+            reason
+        }, message ? {
+            message
+        } : {})
+    });
+function only(view, fieldIds) {
+    return Object.fromEntries(fieldIds.map((fieldId)=>{
+        var _view_fieldId;
+        return [
+            fieldId,
+            (_view_fieldId = view[fieldId]) != null ? _view_fieldId : null
+        ];
+    }));
+}
+/** Updates: each record's values with the row's changes over them, in one batch. */ async function applyUpdates(deps, dataset, rows) {
+    if (!rows.length) return [];
+    const { model, records } = dataset;
+    const docs = await getAll(deps.firestore, rows.map((row)=>records.doc(row.recordId)));
+    const batch = deps.firestore.batch();
+    const landed = [];
+    let writes = 0;
+    rows.forEach((row, at)=>{
+        var _doc_get;
+        const doc = docs[at];
+        if (!doc.exists) return landed.push(failed(row, 'matchedRecordMissing'));
+        const stored = _extends({}, (_doc_get = doc.get('values')) != null ? _doc_get : {});
+        const before = viewOf(model, doc);
+        const { values: changes, cleared } = datasetStorageValues(model, Object.fromEntries(row.diff.map((change)=>[
+                change.fieldId,
+                change.after
+            ])));
+        const merged = _extends({}, stored, changes);
+        for (const fieldId of cleared)delete merged[fieldId];
+        const coerced = fillRecordAddresses(model, merged);
+        const errors = datasetWriteErrors(model, coerced, new Set([
+            ...Object.keys(changes),
+            ...cleared
+        ]));
+        if (Object.keys(errors).length) return landed.push(failed(row, 'refusedValue', describeDatasetRecordErrors(errors)));
+        batch.set(doc.ref, _extends({
+            values: coerced
+        }, datasetIntegrityUpdate(model, coerced, FieldValue.delete()), {
+            updatedAt: Timestamp.now()
+        }), // Each named field replaced whole: `merge: true` would fold the new
+        // `filterValues` into the stored map and keep a cleared value.
+        {
+            mergeFields: RECORD_MERGE_FIELDS
+        });
+        writes += 1;
+        const touched = row.diff.map((change)=>change.fieldId);
+        const after = datasetRecordTransferValues(model, {
+            values: coerced
+        });
+        landed.push({
+            result: {
+                row: row.index,
+                outcome: 'updated',
+                recordId: doc.id
+            },
+            undo: {
+                row: row.index,
+                recordId: doc.id,
+                action: 'updated',
+                previous: only(before, touched),
+                written: only(after, touched),
+                modes: Object.fromEntries(row.diff.map((change)=>[
+                        change.fieldId,
+                        change.mode
+                    ]))
+            }
+        });
+        return undefined;
+    });
+    if (writes) await batch.commit();
+    return landed;
+}
+/**
+ * New records written as `/api/orgs/datasets` creates them: the data storage
+ * band refusing first, then counted, decided and written in ONE transaction
+ * against `recordsPerDataset`, each with the integrity index of its values.
+ * A record already there is left as it is. `values` are validated already.
+ */ async function writeNewRecords(deps, dataset, entries) {
+    const { model, records } = dataset;
+    const org = (await dataset.orgRef.get()).data();
+    // Bytes, not rows (AGL-2163), before the write as on every other create.
+    const storage = await dataStorageRefusal(org, dataset.orgRef);
+    if (storage) return {
+        message: `Dataset storage is full (${storage.includedMb} MB on this plan) — upgrade in Billing`
+    };
+    const outcomes = await deps.firestore.runTransaction(async (tx)=>{
+        var _data_count;
+        const refs = entries.map((entry)=>records.doc(entry.id));
+        const existing = await tx.getAll(...refs);
+        const live = Number((_data_count = (await tx.get(records.count())).data().count) != null ? _data_count : 0);
+        let room = checkQuota(org, 'recordsPerDataset', live).remaining;
+        let order = live;
+        return entries.map((entry, at)=>{
+            if (existing[at].exists) return 'existed';
+            if (room <= 0) return 'refused';
+            room -= 1;
+            tx.create(refs[at], _extends({
+                values: entry.values
+            }, datasetIntegrityFields(model, entry.values), {
+                order: order++,
+                createdAt: Timestamp.now(),
+                updatedAt: Timestamp.now()
+            }));
+            return 'created';
+        });
+    });
+    const limit = checkQuota(org, 'recordsPerDataset', 0).limit;
+    return {
+        outcomes,
+        limitMessage: `Record limit reached (${limit}) — upgrade in Billing`
+    };
+}
+/**
+ * Creates, through {@link writeNewRecords}: validated first, so a row the
+ * model refuses fails naming why. A row the plan's room has run out for
+ * fails as past the plan; a record this job already created (a retried
+ * chunk) is reported, not written again.
+ */ async function applyCreates(deps, ctx, dataset, rows) {
+    if (!rows.length) return [];
+    const { model } = dataset;
+    const landed = [];
+    const prepared = [];
+    for (const row of rows){
+        var _ctx_jobId;
+        const { values } = datasetStorageValues(model, Object.fromEntries(row.diff.map((change)=>[
+                change.fieldId,
+                change.after
+            ])));
+        const coerced = fillRecordAddresses(model, values);
+        const errors = datasetWriteErrors(model, coerced);
+        if (Object.keys(errors).length) landed.push(failed(row, 'refusedValue', describeDatasetRecordErrors(errors)));
+        else prepared.push({
+            row,
+            id: transferRecordId((_ctx_jobId = ctx.jobId) != null ? _ctx_jobId : ctx.resource, row.index),
+            values: coerced
+        });
+    }
+    if (!prepared.length) return landed;
+    const written = await writeNewRecords(deps, dataset, prepared);
+    if ('message' in written) return [
+        ...landed,
+        ...prepared.map((entry)=>failed(entry.row, 'planLimit', written.message))
+    ];
+    prepared.forEach((entry, at)=>{
+        if (written.outcomes[at] === 'refused') {
+            landed.push(failed(entry.row, 'planLimit', written.limitMessage));
+            return;
+        }
+        const values = datasetRecordTransferValues(model, {
+            values: entry.values
+        });
+        delete values[TRANSFER_ID_FIELD];
+        landed.push({
+            result: {
+                row: entry.row.index,
+                outcome: 'created',
+                recordId: entry.id
+            },
+            undo: {
+                row: entry.row.index,
+                recordId: entry.id,
+                action: 'created',
+                written: values
+            }
+        });
+    });
+    return landed;
+}
+/*------------------------------------------
+ * References the person chose to create
+ *-----------------------------------------*/ /**
+ * The id a record created for a reference's name is given: ten characters
+ * of the resource-id alphabet, from the job, the target, the field it is
+ * named in and the name as the engine keys it — so every row naming it, in
+ * any chunk, and a retried chunk all land on the one record.
+ */ export function transferReferenceRecordId(jobId, datasetId, displayFieldId, name) {
+    return transferRecordId(`${jobId}:${datasetId}:${displayFieldId}:${transferLookupKey(name)}`, 0);
+}
+/** The names a reference value asks to create, by the engine's marker. */ function namesToCreate(value) {
+    const names = [];
+    for (const item of Array.isArray(value) ? value : [
+        value
+    ]){
+        const name = transferLookupNewName(item);
+        if (name) names.push(name);
+    }
+    return names;
+}
+function namedReferences(model) {
+    var _model_order;
+    const out = [];
+    for (const fieldId of (_model_order = model.order) != null ? _model_order : []){
+        var _model_fields, _field_reference, _field_reference1;
+        const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[fieldId];
+        const targetId = (field == null ? void 0 : field.type) === 'reference' ? (_field_reference = field.reference) == null ? void 0 : _field_reference.datasetId : undefined;
+        const displayFieldId = field == null ? void 0 : (_field_reference1 = field.reference) == null ? void 0 : _field_reference1.displayFieldId;
+        if (targetId && displayFieldId) out.push({
+            transferId: datasetTransferFieldId(fieldId),
+            targetId,
+            displayFieldId
+        });
+    }
+    return out;
+}
+/**
+ * The records the person chose to create from a reference's names, created
+ * in the referenced dataset before the rows that name them are written: each
+ * name once, holding the name in the field the reference displays, held to
+ * the target's model and its plan, through {@link writeNewRecords}. The rows
+ * come back naming the records' ids; a row naming one that could not be
+ * created fails, saying why.
+ */ async function createNamedReferences(deps, ctx, dataset, rows) {
+    var _ctx_jobId;
+    const references = namedReferences(dataset.model);
+    // Target and display field → name key → the name as the file spelled it.
+    const wanted = new Map();
+    for (const row of rows){
+        for (const change of row.diff){
+            var _wanted_get;
+            const reference = references.find((one)=>one.transferId === change.fieldId);
+            const names = reference ? namesToCreate(change.after) : [];
+            if (!reference || !names.length) continue;
+            const at = `${reference.targetId}\u0000${reference.displayFieldId}`;
+            const group = (_wanted_get = wanted.get(at)) != null ? _wanted_get : {
+                reference,
+                names: new Map()
+            };
+            for (const name of names)if (!group.names.has(transferLookupKey(name))) group.names.set(transferLookupKey(name), name);
+            wanted.set(at, group);
+        }
+    }
+    if (!wanted.size) return {
+        rows: [
+            ...rows
+        ],
+        failed: []
+    };
+    // Name key → the record's id, or why it could not be created; per target and display field.
+    const outcome = new Map();
+    const jobId = (_ctx_jobId = ctx.jobId) != null ? _ctx_jobId : ctx.resource;
+    for (const [at, { reference, names }] of wanted){
+        const decided = new Map();
+        outcome.set(at, decided);
+        const target = await datasetFor(deps, targetContext(ctx, reference.targetId));
+        await ensureDeclaredCustomFieldTypes(target.model);
+        const entries = [];
+        for (const [key, name] of names){
+            const values = fillRecordAddresses(target.model, datasetStorageValues(target.model, {
+                [datasetTransferFieldId(reference.displayFieldId)]: name
+            }).values);
+            const errors = datasetWriteErrors(target.model, values);
+            if (Object.keys(errors).length) {
+                decided.set(key, {
+                    message: `“${name}” could not be created in ${target.label}: ${describeDatasetRecordErrors(errors)}`
+                });
+            } else {
+                entries.push({
+                    key,
+                    id: transferReferenceRecordId(jobId, target.id, reference.displayFieldId, name),
+                    values
+                });
+            }
+        }
+        if (!entries.length) continue;
+        const written = await writeNewRecords(deps, target, entries);
+        entries.forEach((entry, index)=>{
+            const name = names.get(entry.key);
+            if ('message' in written) decided.set(entry.key, {
+                message: `“${name}” could not be created in ${target.label}: ${written.message}`
+            });
+            else if (written.outcomes[index] === 'refused') {
+                decided.set(entry.key, {
+                    message: `“${name}” could not be created in ${target.label}: ${written.limitMessage}`
+                });
+            } else decided.set(entry.key, {
+                id: entry.id
+            });
+        });
+        if (!('message' in written) && written.outcomes.includes('created')) {
+            await announceDatasetRecords({
+                firestore: deps.firestore,
+                orgId: ctx.orgId,
+                datasetId: target.id
+            });
+        }
+    }
+    const out = [];
+    const refused = [];
+    for (const row of rows){
+        const problems = [];
+        const diff = row.diff.map((change)=>{
+            const reference = references.find((one)=>one.transferId === change.fieldId);
+            if (!reference || !namesToCreate(change.after).length) return change;
+            const decided = outcome.get(`${reference.targetId}\u0000${reference.displayFieldId}`);
+            const idOf = (item)=>{
+                const name = transferLookupNewName(item);
+                if (!name) return item;
+                const one = decided == null ? void 0 : decided.get(transferLookupKey(name));
+                if (one && 'id' in one) return one.id;
+                problems.push(one && 'message' in one ? one.message : `“${name}” could not be created.`);
+                return item;
+            };
+            return _extends({}, change, {
+                after: Array.isArray(change.after) ? [
+                    ...new Set(change.after.map(idOf))
+                ] : idOf(change.after)
+            });
+        });
+        if (problems.length) refused.push(failed(row, 'refusedValue', [
+            ...new Set(problems)
+        ].join(' ')));
+        else out.push(_extends({}, row, {
+            diff
+        }));
+    }
+    return {
+        rows: out,
+        failed: refused
+    };
+}
+export async function applyDatasetChunk(deps, ctx, chunk, writer) {
+    const dataset = await datasetFor(deps, ctx);
+    // A plugin's field validator runs only once its plugin registered it.
+    await ensureDeclaredCustomFieldTypes(dataset.model);
+    const results = [];
+    const undo = [];
+    const pending = [];
+    for (const row of chunk.rows){
+        const done = await writer.alreadyApplied(row.index);
+        if (done) results.push(done);
+        else pending.push(row);
+    }
+    // The records the rows' references name for creation, before the rows.
+    const named = await createNamedReferences(deps, ctx, dataset, pending);
+    const landed = [
+        ...named.failed,
+        ...await applyUpdates(deps, dataset, named.rows.filter((row)=>row.verdict === 'update' && row.recordId)),
+        ...await applyCreates(deps, ctx, dataset, named.rows.filter((row)=>row.verdict === 'create'))
+    ];
+    for (const entry of landed){
+        await writer.markApplied(entry.result, entry.undo);
+        results.push(entry.result);
+        if (entry.undo) undo.push(entry.undo);
+    }
+    // The pages repeating over the dataset, told once for the chunk (AGL-3113).
+    if (undo.length) {
+        await announceDatasetRecords({
+            firestore: deps.firestore,
+            orgId: ctx.orgId,
+            datasetId: dataset.id
+        });
+    }
+    return {
+        results,
+        undo
+    };
+}
+/*------------------------------------------
+ * Undo
+ *-----------------------------------------*/ /**
+ * Deletes a record the import created, keeping every reference to it whole
+ * (see the block header). `false` when a `restrict` reference holds it.
+ */ async function deleteWithIntegrity(deps, dataset, recordId) {
+    const others = await dataset.orgRef.collection('datasets').get();
+    const strips = [];
+    const batch = deps.firestore.batch();
+    for (const other of others.docs){
+        var _model_fields;
+        const model = effectiveDatasetModel(other.data());
+        const referencing = Object.keys((_model_fields = model.fields) != null ? _model_fields : {}).filter((fieldId)=>{
+            var _model_fields_fieldId, _model_fields_fieldId_reference, _model_fields_fieldId1;
+            return ((_model_fields_fieldId = model.fields[fieldId]) == null ? void 0 : _model_fields_fieldId.type) === 'reference' && ((_model_fields_fieldId1 = model.fields[fieldId]) == null ? void 0 : (_model_fields_fieldId_reference = _model_fields_fieldId1.reference) == null ? void 0 : _model_fields_fieldId_reference.datasetId) === dataset.id;
+        });
+        if (!referencing.length) continue;
+        const holders = (await other.ref.collection('records').where('referencedIds', 'array-contains', recordId).get()).docs.filter((doc)=>referencing.some((fieldId)=>{
+                var _doc_get;
+                const stored = ((_doc_get = doc.get('values')) != null ? _doc_get : {})[fieldId];
+                return Array.isArray(stored) ? stored.includes(recordId) : stored === recordId;
+            }));
+        if (!holders.length) continue;
+        if (referencing.some((fieldId)=>{
+            var _model_fields_fieldId_reference, _model_fields_fieldId;
+            return ((_model_fields_fieldId = model.fields[fieldId]) == null ? void 0 : (_model_fields_fieldId_reference = _model_fields_fieldId.reference) == null ? void 0 : _model_fields_fieldId_reference.onDelete) === 'restrict';
+        })) return false;
+        for (const holder of holders){
+            strips.push(()=>{
+                var _holder_get;
+                const values = _extends({}, (_holder_get = holder.get('values')) != null ? _holder_get : {});
+                for (const fieldId of referencing){
+                    const stored = values[fieldId];
+                    if (Array.isArray(stored)) values[fieldId] = stored.filter((id)=>id !== recordId);
+                    else if (stored === recordId) delete values[fieldId];
+                }
+                batch.update(holder.ref, _extends({
+                    values
+                }, datasetIntegrityUpdate(model, values, FieldValue.delete()), {
+                    updatedAt: Timestamp.now()
+                }));
+            });
+        }
+    }
+    for (const strip of strips)strip();
+    batch.delete(dataset.records.doc(recordId));
+    await batch.commit();
+    return true;
+}
+export async function revertDatasetChunk(deps, ctx, snapshot, decisions = {}) {
+    const dataset = await datasetFor(deps, ctx);
+    const { model, records } = dataset;
+    const done = [];
+    const conflicts = [];
+    if (!snapshot.entries.length) return {
+        done,
+        conflicts
+    };
+    const docs = await getAll(deps.firestore, snapshot.entries.map((entry)=>records.doc(entry.recordId)));
+    const restores = deps.firestore.batch();
+    let restoring = 0;
+    for (const [at, entry] of snapshot.entries.entries()){
+        var _doc_get;
+        const doc = docs[at];
+        const step = planTransferUndo(entry, doc.exists ? viewOf(model, doc) : null);
+        const revert = step.action !== 'conflict' || decisions[entry.recordId] === 'revert';
+        if (!revert) {
+            conflicts.push(step);
+            continue;
+        }
+        if (step.action === 'nothing') {
+            done.push(step);
+            continue;
+        }
+        if (step.action === 'delete' || step.action === 'conflict' && entry.action === 'created') {
+            if (await deleteWithIntegrity(deps, dataset, entry.recordId)) done.push({
+                action: 'delete',
+                recordId: entry.recordId
+            });
+            else conflicts.push(step);
+            continue;
+        }
+        // Restore (or a conflict the person chose to revert): the fields the
+        // import wrote, back to what they held before it.
+        const stored = _extends({}, (_doc_get = doc.get('values')) != null ? _doc_get : {});
+        const { values, cleared } = datasetStorageValues(model, step.values);
+        const merged = _extends({}, stored, values);
+        for (const fieldId of cleared)delete merged[fieldId];
+        restores.set(doc.ref, _extends({
+            values: merged
+        }, datasetIntegrityUpdate(model, merged, FieldValue.delete()), {
+            updatedAt: Timestamp.now()
+        }), {
+            mergeFields: RECORD_MERGE_FIELDS
+        });
+        restoring += 1;
+        done.push({
+            action: 'restore',
+            recordId: entry.recordId,
+            values: step.values
+        });
+    }
+    if (restoring) await restores.commit();
+    if (done.some((step)=>step.action !== 'nothing')) {
+        await announceDatasetRecords({
+            firestore: deps.firestore,
+            orgId: ctx.orgId,
+            datasetId: dataset.id
+        });
+    }
+    return {
+        done,
+        conflicts
+    };
+}
+/*------------------------------------------
+ * The hooks
+ *-----------------------------------------*/ /** The records hooks for `data.dataset`, over `deps` (the Admin SDK's Firestore by default). */ export function datasetTransferHooks(deps = productionDeps) {
+    return {
+        fields: async (ctx)=>{
+            const dataset = await datasetFor(deps(), ctx);
+            return datasetTransferCatalog(dataset.model, dataset.label, await datasetReferenceTargets(deps(), ctx, dataset.model));
+        },
+        matchKeys: async (ctx)=>datasetMatchKeyOffer((await datasetFor(deps(), ctx)).model),
+        count: (ctx, options)=>countDatasetRecords(deps(), ctx, options),
+        readPage: (ctx, cursor, fieldIds, options)=>readDatasetPage(deps(), ctx, cursor, fieldIds, options),
+        lookup: (ctx, requests)=>lookupDatasetRecords(deps(), ctx, requests),
+        suggest: (ctx, request)=>suggestDatasetRecords(deps(), ctx, request),
+        picklists: (ctx, picklistIds)=>datasetPicklists(deps(), ctx, picklistIds),
+        addPicklistValues: (ctx, picklistId, values)=>addDatasetOptions(deps(), ctx, picklistId, values),
+        plan: async (ctx, input)=>{
+            const dataset = await datasetFor(deps(), ctx);
+            await ensureDeclaredCustomFieldTypes(dataset.model);
+            const plan = refuseInvalidDatasetRows(dataset.model, buildTransferPlan(input), input.existing);
+            return limitDatasetCreates(plan, (await createLimits(dataset)).maxCreates);
+        },
+        apply: (ctx, chunk, writer)=>applyDatasetChunk(deps(), ctx, chunk, writer),
+        revert: (ctx, snapshot, decisions)=>revertDatasetChunk(deps(), ctx, snapshot, decisions)
+    };
+}
+
+//# sourceMappingURL=dataset-transfer.server.js.map