@aglyn/plugins-data 1.0.0-beta.220 → 1.0.0-beta.222

package/src/lib/model/dataset-models.js

@@ -0,0 +1,806 @@
+/**
+ * @license
+ * Copyright 2026 Aglyn LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */ import { _ as _extends } from "@swc/helpers/_/_extends";
+import { validateCustomFieldValue } from "@aglyn/aglyn/plugin-manager/custom-fields";
+/** Display fallback for raw ids: "roast_preference" → "Roast preference". */ export function humanizeDatasetFieldId(id) {
+    const words = id.replace(/_/g, ' ').trim();
+    return words ? words.charAt(0).toUpperCase() + words.slice(1) : id;
+}
+export const DATASET_FIELD_TYPES = [
+    'text',
+    'bool',
+    'int32',
+    'int64',
+    'float',
+    'timestamp',
+    'coordinates',
+    'map',
+    'sorted',
+    'bytes',
+    'nil',
+    'reference'
+];
+/** Display names, mirroring dod.ts `lbl`. */ export const DATASET_FIELD_TYPE_LABELS = {
+    text: 'Text',
+    bool: 'Boolean',
+    int32: 'Integer',
+    int64: 'Big integer',
+    float: 'Number',
+    timestamp: 'Date & time',
+    coordinates: 'Coordinates',
+    map: 'Map',
+    sorted: 'List',
+    bytes: 'Bytes',
+    nil: 'Null',
+    reference: 'Reference'
+};
+/**
+ * Migration shim (v1 → v2): datasets created before models carry flat
+ * `fields: string[]` — derive a model where every column is an optional
+ * text field whose id is the column name, so the editor and AGL-103
+ * bindings keep working on unmigrated docs during rollout.
+ */ export function deriveModelFromFields(fields) {
+    const model = {
+        fields: {},
+        order: []
+    };
+    for (const name of fields){
+        // Raw snake_case ids read as titles in table headers and pickers;
+        // the id stays the stable key (AGL-558).
+        model.fields[name] = {
+            name: humanizeDatasetFieldId(name),
+            type: 'text'
+        };
+        model.order.push(name);
+    }
+    return model;
+}
+/**
+ * Model from quick-creator entries (AGL-558): stable slug ids with the
+ * human display names preserved — "Roast preference" → id
+ * `roast_preference`, name "Roast preference".
+ */ export function modelFromFieldEntries(entries) {
+    const model = {
+        fields: {},
+        order: []
+    };
+    for (const entry of entries){
+        model.fields[entry.id] = {
+            name: entry.name,
+            type: 'text'
+        };
+        model.order.push(entry.id);
+    }
+    return model;
+}
+/**
+ * The part of a dataset's model a repeat reads (AGL-180): each reference
+ * field, by the id of the dataset it points into, which is a key the rows map
+ * answers under. Every field the model declares, in `order` or not — a hop
+ * resolves wherever its target's rows were loaded.
+ */ export function repeatRowsModelOf(model) {
+    const references = {};
+    for (const [fieldId, field] of Object.entries(model.fields)){
+        var _field_reference;
+        const target = (field == null ? void 0 : field.type) === 'reference' ? (_field_reference = field.reference) == null ? void 0 : _field_reference.datasetId : undefined;
+        if (target) references[fieldId] = target;
+    }
+    return {
+        references
+    };
+}
+/** The dataset's model, deriving one from v1 `fields` when absent. */ export function effectiveDatasetModel(dataset) {
+    var _dataset_fields;
+    var _dataset_model, _dataset_model_order;
+    if (((_dataset_model = dataset.model) == null ? void 0 : _dataset_model.fields) && ((_dataset_model_order = dataset.model.order) == null ? void 0 : _dataset_model_order.length)) {
+        return dataset.model;
+    }
+    return deriveModelFromFields((_dataset_fields = dataset.fields) != null ? _dataset_fields : []);
+}
+const isFiniteNumber = (value)=>typeof value === 'number' && Number.isFinite(value);
+/**
+ * Validates a document's values against a model, returning fieldId →
+ * human-readable error (empty object = valid). Values are expected in
+ * storage form (see the module doc); use `coerceDocumentValues` first
+ * when starting from user input strings.
+ */ export function validateDocument(model, values) {
+    const errors = {};
+    for (const fieldId of model.order){
+        var _field_validation;
+        var _field_validation1;
+        const field = model.fields[fieldId];
+        if (!field) continue;
+        const value = values[fieldId];
+        const required = field.required || ((_field_validation1 = field.validation) == null ? void 0 : _field_validation1.required);
+        // An empty list holds nothing, so it answers "required" as an empty
+        // string does: `,,` typed into a list field coerces to `[]` (AGL-3496).
+        const empty = value == null || value === '' || Array.isArray(value) && !value.length && isListField(field);
+        if (empty) {
+            if (required) errors[fieldId] = `${field.name} is required`;
+            continue;
+        }
+        const rules = (_field_validation = field.validation) != null ? _field_validation : {};
+        switch(field.type){
+            case 'text':
+                {
+                    var _rules_options;
+                    if (typeof value !== 'string') {
+                        errors[fieldId] = `${field.name} must be text`;
+                        break;
+                    }
+                    if (rules.min != null && value.length < rules.min) {
+                        errors[fieldId] = `${field.name} must be at least ${rules.min} characters`;
+                    } else if (rules.max != null && value.length > rules.max) {
+                        errors[fieldId] = `${field.name} must be at most ${rules.max} characters`;
+                    } else if (rules.regex) {
+                        try {
+                            if (!new RegExp(rules.regex).test(value)) {
+                                errors[fieldId] = `${field.name} has an invalid format`;
+                            }
+                        } catch (unused) {
+                        // Broken pattern in the model never blocks writes.
+                        }
+                    }
+                    if (!errors[fieldId] && ((_rules_options = rules.options) == null ? void 0 : _rules_options.length) && !rules.options.includes(value)) {
+                        errors[fieldId] = `${field.name} must be one of: ${rules.options.join(', ')}`;
+                    }
+                    break;
+                }
+            case 'bool':
+                if (typeof value !== 'boolean') {
+                    errors[fieldId] = `${field.name} must be true or false`;
+                }
+                break;
+            case 'int32':
+            case 'int64':
+                if (!isFiniteNumber(value) || !Number.isInteger(value)) {
+                    errors[fieldId] = `${field.name} must be a whole number`;
+                    break;
+                }
+                if (rules.min != null && value < rules.min) {
+                    errors[fieldId] = `${field.name} must be ≥ ${rules.min}`;
+                } else if (rules.max != null && value > rules.max) {
+                    errors[fieldId] = `${field.name} must be ≤ ${rules.max}`;
+                }
+                break;
+            case 'float':
+            case 'timestamp':
+                if (!isFiniteNumber(value)) {
+                    errors[fieldId] = field.type === 'timestamp' ? `${field.name} must be a date` : `${field.name} must be a number`;
+                    break;
+                }
+                if (rules.min != null && value < rules.min) {
+                    errors[fieldId] = `${field.name} must be ≥ ${rules.min}`;
+                } else if (rules.max != null && value > rules.max) {
+                    errors[fieldId] = `${field.name} must be ≤ ${rules.max}`;
+                }
+                break;
+            case 'coordinates':
+                {
+                    const coordinates = value;
+                    if (!isFiniteNumber(coordinates == null ? void 0 : coordinates.latitude) || !isFiniteNumber(coordinates == null ? void 0 : coordinates.longitude) || Math.abs(coordinates.latitude) > 90 || Math.abs(coordinates.longitude) > 180) {
+                        errors[fieldId] = `${field.name} must be valid coordinates`;
+                    }
+                    break;
+                }
+            case 'sorted':
+                // A list of values: an entry that is itself an object or a list is
+                // not one, and `coerceListValue` leaves it here to be refused.
+                if (!Array.isArray(value) || value.some((entry)=>entry !== null && typeof entry === 'object')) {
+                    errors[fieldId] = `${field.name} must be a list`;
+                }
+                break;
+            case 'map':
+                if (typeof value !== 'object' || Array.isArray(value)) {
+                    errors[fieldId] = `${field.name} must be a map`;
+                }
+                break;
+            case 'reference':
+                var _field_reference;
+                if ((_field_reference = field.reference) == null ? void 0 : _field_reference.multiple) {
+                    if (!Array.isArray(value) || value.some((entry)=>typeof entry !== 'string' || !entry)) {
+                        errors[fieldId] = `${field.name} must reference documents`;
+                    }
+                } else if (typeof value !== 'string' || !value) {
+                    errors[fieldId] = `${field.name} must reference a document`;
+                }
+                break;
+            case 'bytes':
+            case 'nil':
+                break;
+        }
+    }
+    // Custom field types (AGL-434): plugin validators run after the base
+    // checks, only on fields that passed them.
+    for (const fieldId of model.order){
+        const field = model.fields[fieldId];
+        if (!(field == null ? void 0 : field.customType) || errors[fieldId]) continue;
+        const customError = validateCustomFieldValue(field.customType, values[fieldId]);
+        if (customError) errors[fieldId] = `${field.name}: ${customError}`;
+    }
+    return errors;
+}
+/** Storage value → grid display string (AGL-179). */ export function formatDatasetValue(field, value) {
+    if (value == null || value === '') return '';
+    switch(field.type){
+        case 'bool':
+            return value === true ? '✓' : value === false ? '—' : String(value);
+        case 'timestamp':
+            return isFiniteNumber(value) ? new Date(value).toISOString().slice(0, 16).replace('T', ' ') : String(value);
+        case 'coordinates':
+            {
+                const coordinates = value;
+                return isFiniteNumber(coordinates == null ? void 0 : coordinates.latitude) && isFiniteNumber(coordinates == null ? void 0 : coordinates.longitude) ? `${coordinates.latitude}, ${coordinates.longitude}` : String(value);
+            }
+        case 'sorted':
+            return Array.isArray(value) ? value.join(', ') : String(value);
+        case 'map':
+            try {
+                return JSON.stringify(value);
+            } catch (unused) {
+                return String(value);
+            }
+        default:
+            return String(value);
+    }
+}
+/**
+ * Storage value → form-input string (AGL-179): the inverse of
+ * `coerceDocumentValues` for populating type-appropriate inputs
+ * (timestamps as `datetime-local` values, coordinates as "lat, lon").
+ */ export function datasetValueToInput(field, value) {
+    if (value == null) return '';
+    if (field.type === 'timestamp' && isFiniteNumber(value)) {
+        return new Date(value).toISOString().slice(0, 16);
+    }
+    if (field.type === 'bool') {
+        return value === true ? 'true' : value === false ? 'false' : String(value);
+    }
+    if (field.type === 'coordinates' || field.type === 'sorted') {
+        return formatDatasetValue(field, value);
+    }
+    if (field.type === 'map') return formatDatasetValue(field, value);
+    return String(value);
+}
+/**
+ * Whether a field stores a list: `sorted`, and a `reference` that may point at
+ * more than one record. Both go through {@link coerceListValue}.
+ */ export function isListField(field) {
+    var _field_reference;
+    return field.type === 'sorted' || field.type === 'reference' && Boolean((_field_reference = field.reference) == null ? void 0 : _field_reference.multiple);
+}
+/**
+ * The entries of a list given as an array, each a trimmed string with the
+ * empty ones dropped — or `null` when an entry is not a scalar (an object, a
+ * nested list), which is not a list of values and is left for
+ * `validateDocument` to refuse rather than flattened into `[object Object]`.
+ */ function listEntries(input) {
+    const entries = [];
+    for (const entry of input){
+        if (entry == null) continue;
+        if (typeof entry !== 'string' && typeof entry !== 'number' && typeof entry !== 'boolean') {
+            return null;
+        }
+        const text = String(entry).trim();
+        if (text) entries.push(text);
+    }
+    return entries;
+}
+/**
+ * A list field's value in storage form (AGL-3496) — the ONE coercion every
+ * writer of a `sorted` or multiple-`reference` field reaches through
+ * {@link coerceDocumentValues}: the console's record form and its CSV/JSON
+ * import, `/api/orgs/datasets`, `/v1` record create and update, form
+ * submissions and workflow steps (`prepareDatasetRecordWrite`).
+ *
+ * - An array is taken as the list: entries stringified and trimmed, empty
+ *   ones dropped.
+ * - A string that parses as a JSON array of scalars is that array. A JSON
+ *   import hands a list cell over as `["Residential","Commercial"]`, and so
+ *   does any API client that serialized one; split on commas, it was stored
+ *   as `["[\"Residential\"", "\"Commercial\"]"]` and no filter matched it.
+ *   The console form round-trips such a record too: it shows the stored
+ *   fragments joined, which reads back as the JSON array, so re-saving a
+ *   record written that way repairs it.
+ * - Any other string is comma-separated, as typed into the form.
+ * - Anything else (a number, an object, an array holding objects) passes
+ *   through untouched for `validateDocument` to refuse.
+ */ export function coerceListValue(raw) {
+    var _listEntries;
+    if (Array.isArray(raw)) return (_listEntries = listEntries(raw)) != null ? _listEntries : raw;
+    if (typeof raw !== 'string') return raw;
+    const text = raw.trim();
+    if (text.startsWith('[') && text.endsWith(']')) {
+        try {
+            const parsed = JSON.parse(text);
+            const entries = Array.isArray(parsed) ? listEntries(parsed) : null;
+            if (entries) return entries;
+        } catch (unused) {
+        // Bracketed but not JSON — `[draft], final` — is a comma list.
+        }
+    }
+    return text.split(',').map((part)=>part.trim()).filter(Boolean);
+}
+const BOOL_TRUE_WORDS = new Set([
+    'true',
+    'yes',
+    'on',
+    '1'
+]);
+const BOOL_FALSE_WORDS = new Set([
+    'false',
+    'no',
+    'off',
+    '0'
+]);
+/**
+ * Coerces user-input strings (form fields, CSV cells) into storage form
+ * per the field type. Unparseable input is passed through untouched so
+ * `validateDocument` reports it instead of silently mangling it.
+ *
+ * A list field is coerced whatever shape it arrives in — array or string —
+ * through {@link coerceListValue}; every other non-string value is already
+ * in storage form and passes through.
+ */ export function coerceDocumentValues(model, input) {
+    const values = {};
+    for (const fieldId of model.order){
+        const field = model.fields[fieldId];
+        const raw = input[fieldId];
+        if (!field || raw == null || raw === '') continue;
+        if (isListField(field)) {
+            values[fieldId] = coerceListValue(raw);
+            continue;
+        }
+        if (typeof raw !== 'string') {
+            values[fieldId] = raw;
+            continue;
+        }
+        switch(field.type){
+            case 'bool':
+                {
+                    // The words a form control or a CSV cell says yes and no with: a
+                    // lone checkbox posts `on`, a spreadsheet exports `TRUE` or `1`.
+                    const word = raw.trim().toLowerCase();
+                    values[fieldId] = BOOL_TRUE_WORDS.has(word) ? true : BOOL_FALSE_WORDS.has(word) ? false : raw;
+                    break;
+                }
+            case 'int32':
+            case 'int64':
+                {
+                    const parsed = Number(raw);
+                    values[fieldId] = Number.isInteger(parsed) ? parsed : raw;
+                    break;
+                }
+            case 'float':
+                {
+                    const parsed = Number(raw);
+                    values[fieldId] = Number.isFinite(parsed) ? parsed : raw;
+                    break;
+                }
+            case 'timestamp':
+                {
+                    const millis = Date.parse(raw);
+                    values[fieldId] = Number.isFinite(millis) ? millis : raw;
+                    break;
+                }
+            case 'coordinates':
+                {
+                    const [latitude, longitude] = raw.split(',').map((part)=>Number(part.trim()));
+                    values[fieldId] = Number.isFinite(latitude) && Number.isFinite(longitude) ? {
+                        latitude,
+                        longitude
+                    } : raw;
+                    break;
+                }
+            case 'map':
+                try {
+                    values[fieldId] = JSON.parse(raw);
+                } catch (unused) {
+                    values[fieldId] = raw;
+                }
+                break;
+            default:
+                values[fieldId] = raw;
+        }
+    }
+    return values;
+}
+/**
+ * Every reference id a record holds, flattened out of its `reference` fields
+ * into one de-duplicated list.
+ *
+ * A reference field stores either a single target id or — when
+ * `reference.multiple` is set — an array of them, and a model may declare
+ * several. Both shapes flatten into the same list because the question asked
+ * of it is never "which field", it is "does this record still point at that
+ * document".
+ *
+ * Fields are read from `order` UNIONED with the keys of `fields`. Every other
+ * pass over a model walks `order` alone, which is right for anything the user
+ * sees — but a reference declared without a display slot still holds a real
+ * FKey, and an integrity index that skipped it would under-report.
+ */ export function datasetReferencedIds(model, values) {
+    var _model_order, _model_fields;
+    const ids = new Set();
+    const fieldIds = new Set([
+        ...(_model_order = model.order) != null ? _model_order : [],
+        ...Object.keys((_model_fields = model.fields) != null ? _model_fields : {})
+    ]);
+    for (const fieldId of fieldIds){
+        var _model_fields_fieldId, _model_fields1;
+        if (((_model_fields1 = model.fields) == null ? void 0 : (_model_fields_fieldId = _model_fields1[fieldId]) == null ? void 0 : _model_fields_fieldId.type) !== 'reference') continue;
+        const stored = values == null ? void 0 : values[fieldId];
+        for (const entry of Array.isArray(stored) ? stored : [
+            stored
+        ]){
+            const id = typeof entry === 'string' ? entry.trim() : '';
+            if (id) ids.add(id);
+        }
+    }
+    return [
+        ...ids
+    ];
+}
+/**
+ * The integrity index a record write carries, so referential integrity can be
+ * decided by a QUERY rather than by the rows a listener happened to fetch.
+ *
+ * Deleting a record has to answer "is anything still pointing at this?", and
+ * `restrict` refuses the delete on a yes while `setNull` strips the FKey. That
+ * question cannot be asked where the answer is stored: dataset fields are
+ * user-defined, so `records.values` carries a deliberate index exemption in
+ * `cloud/firebase-firestore.indexes.json` — auto-indexing an unbounded map
+ * blows Firestore's per-document index-entry limit — and
+ * `where('values.<fieldId>', '==', id)` is therefore FAILED_PRECONDITION.
+ * Reading a page of records and filtering them in the browser answers only for
+ * the rows in that page, which is a browse window standing in for a
+ * correctness check: a reference held outside it reads as no reference at all,
+ * the delete proceeds, and `restrict` silently fails to restrict.
+ *
+ * `referencedIds` is that same reference, denormalized to a top-level array
+ * Firestore will index, so one `array-contains` reaches every record.
+ *
+ * ⚠️ OMITTED when a record references nothing, rather than written as `[]`.
+ * `isNotEmpty` is served as `!= null` and an empty array is not null, so a
+ * record stamped `[]` answers "holds a reference" for a dataset that holds
+ * none.
+ *
+ * ⚠️ Spread this at EVERY write that sets a record's `values`. A write that
+ * sets values without it leaves the index describing the PREVIOUS values, and
+ * a stale integrity index is the exact failure it exists to prevent — with the
+ * UI reporting the record as safe to delete. On a merging write use
+ * `datasetIntegrityUpdate` instead: omission leaves a stale array in place
+ * where clearing the last reference has to remove the field.
+ */ export function datasetIntegrityFields(model, values) {
+    const referencedIds = datasetReferencedIds(model, values);
+    // The records table's filter fields (`datasetFilterKeys`,
+    // `datasetFilterValues`) travel on the same writes for the same reason: a
+    // record written without them is invisible to every filter the query
+    // serves. Omitted when empty, like `referencedIds`.
+    const filterKeys = datasetFilterKeys(model, values);
+    const filterValues = datasetFilterValues(model, values);
+    return _extends({}, referencedIds.length ? {
+        referencedIds
+    } : {}, filterKeys.length ? {
+        filterKeys
+    } : {}, Object.keys(filterValues).length ? {
+        filterValues
+    } : {});
+}
+/**
+ * `datasetIntegrityFields` for a MERGING write (`update`, `set` with
+ * `mergeFields`), where leaving the field out preserves what is stored
+ * rather than clearing it.
+ *
+ * A record whose last reference is cleared has to have the field REMOVED, or
+ * it goes on answering the `array-contains` for a document it no longer points
+ * at — a `restrict` that refuses a delete nothing is holding. `clear` is the
+ * caller's delete sentinel: `deleteField()` in the web SDK,
+ * `FieldValue.delete()` in the Admin SDK. It is passed in rather than imported
+ * because this module is shared by both and must stay SDK-free. The filter
+ * fields are cleared the same way, so a record emptied of every filterable
+ * value stops answering the filters its old values did.
+ *
+ * ⚠️ `filterValues` is a MAP, and it is written WHOLE: `update` and
+ * `mergeFields` replace a field outright, so a value cleared since the last
+ * write leaves the map with it. `set(…, { merge: true })` does NOT — it merges
+ * maps key by key and would keep the cleared value answering its old
+ * equality — so a record write never uses it; list the fields in
+ * `mergeFields` instead.
+ */ export function datasetIntegrityUpdate(model, values, clear) {
+    const referencedIds = datasetReferencedIds(model, values);
+    const filterKeys = datasetFilterKeys(model, values);
+    const filterValues = datasetFilterValues(model, values);
+    return {
+        referencedIds: referencedIds.length ? referencedIds : clear,
+        filterKeys: filterKeys.length ? filterKeys : clear,
+        filterValues: Object.keys(filterValues).length ? filterValues : clear
+    };
+}
+/**
+ * A dataset record's FILTER TOKENS: every word-level question the records
+ * table can ask of a record, flattened into one array of strings Firestore
+ * indexes on its own. Its twin, {@link datasetFilterValues}, holds the
+ * equalities.
+ *
+ * ## Why these fields, and not the values
+ *
+ * A record's fields live under `values`, a user-defined, unbounded map that
+ * the index configuration exempts from indexing — auto-indexing it can exceed
+ * Firestore's per-document index-entry limit — so `where('values.<field>', …)`
+ * is refused. A composite index per dataset would need an admin call per
+ * dataset, runs into the project's composite-index cap, and is deleted by the
+ * next `firestore:indexes` deploy of a file that does not list it.
+ *
+ * `filterKeys` and `filterValues` need none of that. Firestore's AUTOMATIC
+ * single-field indexes serve `where('filterKeys', 'array-contains', token)`
+ * and `where('filterValues.<field>', '==', value)`, and every automatic index
+ * ends in the document name — so any number of those equalities and ONE
+ * `array-contains`, under `orderBy(documentId())`, compose by index merging,
+ * with no composite index and no index deploy, on any dataset of any project
+ * (AGL-3321).
+ *
+ * ## The token grammar
+ *
+ *   `f:<fieldId>=<value>`   a field equals a value. Enum (a text field with
+ *                           `validation.options`) values are kept exactly;
+ *                           plain text is lower-cased, trimmed and clipped to
+ *                           {@link DATASET_FILTER_VALUE_MAX} characters;
+ *                           booleans are `true`/`false`; numbers are
+ *                           `String(number)`; a `sorted` list contributes one
+ *                           token per member.
+ *   `f:<fieldId>^<prefix>`  a word of a plain text field starts with this.
+ *   `s:<prefix>`            a word of ANY text value (enums and list members
+ *                           included) starts with this — the quick search.
+ *
+ * Timestamps, references, maps, bytes, coordinates and nil fields carry no
+ * field tokens: a timestamp is asked in ranges, which one token cannot serve,
+ * and the rest are not asked about by value.
+ *
+ * The array is sorted, de-duplicated and capped at
+ * {@link DATASET_FILTER_KEYS_MAX}. Past the cap, the `=` tokens are kept
+ * first, then the per-field word prefixes, then the search prefixes, so a
+ * very long record loses search reach before it loses exact matches.
+ */ export const DATASET_FILTER_KEYS_MAX = 500;
+/** Plain text values are clipped to this many characters in an `=` token. */ export const DATASET_FILTER_VALUE_MAX = 64;
+/** Word prefixes run from 1 to this many characters. */ export const DATASET_FILTER_PREFIX_MAX = 12;
+/** Only the first this-many words of a value contribute prefixes. */ export const DATASET_FILTER_WORDS_MAX = 40;
+/**
+ * The words of a text value: split on anything that is not a letter or a
+ * digit, lower-cased, at most {@link DATASET_FILTER_WORDS_MAX} of them.
+ */ export function datasetFilterWords(text) {
+    return String(text).toLowerCase().split(RegExp("[^\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376-\\u0377\\u037A-\\u037D\\u037F\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u052F\\u0531-\\u0556\\u0559\\u0560-\\u0588\\u05D0-\\u05EA\\u05EF-\\u05F2\\u0620-\\u064A\\u066E-\\u066F\\u0671-\\u06D3\\u06D5\\u06E5-\\u06E6\\u06EE-\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4-\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u0860-\\u086A\\u0870-\\u0887\\u0889-\\u088F\\u08A0-\\u08C9\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0980\\u0985-\\u098C\\u098F-\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC-\\u09DD\\u09DF-\\u09E1\\u09F0-\\u09F1\\u09FC\\u0A05-\\u0A0A\\u0A0F-\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32-\\u0A33\\u0A35-\\u0A36\\u0A38-\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2-\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0-\\u0AE1\\u0AF9\\u0B05-\\u0B0C\\u0B0F-\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32-\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C-\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99-\\u0B9A\\u0B9C\\u0B9E-\\u0B9F\\u0BA3-\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C39\\u0C3D\\u0C58-\\u0C5A\\u0C5C-\\u0C5D\\u0C60-\\u0C61\\u0C80\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDC-\\u0CDE\\u0CE0-\\u0CE1\\u0CF1-\\u0CF2\\u0D04-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D54-\\u0D56\\u0D5F-\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32-\\u0E33\\u0E40-\\u0E46\\u0E81-\\u0E82\\u0E84\\u0E86-\\u0E8A\\u0E8C-\\u0EA3\\u0EA5\\u0EA7-\\u0EB0\\u0EB2-\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065-\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F5\\u13F8-\\u13FD\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16F1-\\u16F8\\u1700-\\u1711\\u171F-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1878\\u1880-\\u1884\\u1887-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191E\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19B0-\\u19C9\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4C\\u1B83-\\u1BA0\\u1BAE-\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1C80-\\u1C8A\\u1C90-\\u1CBA\\u1CBD-\\u1CBF\\u1CE9-\\u1CEC\\u1CEE-\\u1CF3\\u1CF5-\\u1CF6\\u1CFA\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183-\\u2184\\u2C00-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2-\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3006\\u3031-\\u3035\\u303B-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312F\\u3131-\\u318E\\u31A0-\\u31BF\\u31F0-\\u31FF\\u3400-\\u4DBF\\u4E00-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A-\\uA62B\\uA640-\\uA66E\\uA67F-\\uA69D\\uA6A0-\\uA6E5\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA7DC\\uA7F1-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA8FD-\\uA8FE\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uA9E0-\\uA9E4\\uA9E6-\\uA9EF\\uA9FA-\\uA9FE\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA7E-\\uAAAF\\uAAB1\\uAAB5-\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uAB30-\\uAB5A\\uAB5C-\\uAB69\\uAB70-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40-\\uFB41\\uFB43-\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC\\u{10000}-\\u{1000B}\\u{1000D}-\\u{10026}\\u{10028}-\\u{1003A}\\u{1003C}-\\u{1003D}\\u{1003F}-\\u{1004D}\\u{10050}-\\u{1005D}\\u{10080}-\\u{100FA}\\u{10280}-\\u{1029C}\\u{102A0}-\\u{102D0}\\u{10300}-\\u{1031F}\\u{1032D}-\\u{10340}\\u{10342}-\\u{10349}\\u{10350}-\\u{10375}\\u{10380}-\\u{1039D}\\u{103A0}-\\u{103C3}\\u{103C8}-\\u{103CF}\\u{10400}-\\u{1049D}\\u{104B0}-\\u{104D3}\\u{104D8}-\\u{104FB}\\u{10500}-\\u{10527}\\u{10530}-\\u{10563}\\u{10570}-\\u{1057A}\\u{1057C}-\\u{1058A}\\u{1058C}-\\u{10592}\\u{10594}-\\u{10595}\\u{10597}-\\u{105A1}\\u{105A3}-\\u{105B1}\\u{105B3}-\\u{105B9}\\u{105BB}-\\u{105BC}\\u{105C0}-\\u{105F3}\\u{10600}-\\u{10736}\\u{10740}-\\u{10755}\\u{10760}-\\u{10767}\\u{10780}-\\u{10785}\\u{10787}-\\u{107B0}\\u{107B2}-\\u{107BA}\\u{10800}-\\u{10805}\\u{10808}\\u{1080A}-\\u{10835}\\u{10837}-\\u{10838}\\u{1083C}\\u{1083F}-\\u{10855}\\u{10860}-\\u{10876}\\u{10880}-\\u{1089E}\\u{108E0}-\\u{108F2}\\u{108F4}-\\u{108F5}\\u{10900}-\\u{10915}\\u{10920}-\\u{10939}\\u{10940}-\\u{10959}\\u{10980}-\\u{109B7}\\u{109BE}-\\u{109BF}\\u{10A00}\\u{10A10}-\\u{10A13}\\u{10A15}-\\u{10A17}\\u{10A19}-\\u{10A35}\\u{10A60}-\\u{10A7C}\\u{10A80}-\\u{10A9C}\\u{10AC0}-\\u{10AC7}\\u{10AC9}-\\u{10AE4}\\u{10B00}-\\u{10B35}\\u{10B40}-\\u{10B55}\\u{10B60}-\\u{10B72}\\u{10B80}-\\u{10B91}\\u{10C00}-\\u{10C48}\\u{10C80}-\\u{10CB2}\\u{10CC0}-\\u{10CF2}\\u{10D00}-\\u{10D23}\\u{10D4A}-\\u{10D65}\\u{10D6F}-\\u{10D85}\\u{10E80}-\\u{10EA9}\\u{10EB0}-\\u{10EB1}\\u{10EC2}-\\u{10EC7}\\u{10F00}-\\u{10F1C}\\u{10F27}\\u{10F30}-\\u{10F45}\\u{10F70}-\\u{10F81}\\u{10FB0}-\\u{10FC4}\\u{10FE0}-\\u{10FF6}\\u{11003}-\\u{11037}\\u{11071}-\\u{11072}\\u{11075}\\u{11083}-\\u{110AF}\\u{110D0}-\\u{110E8}\\u{11103}-\\u{11126}\\u{11144}\\u{11147}\\u{11150}-\\u{11172}\\u{11176}\\u{11183}-\\u{111B2}\\u{111C1}-\\u{111C4}\\u{111DA}\\u{111DC}\\u{11200}-\\u{11211}\\u{11213}-\\u{1122B}\\u{1123F}-\\u{11240}\\u{11280}-\\u{11286}\\u{11288}\\u{1128A}-\\u{1128D}\\u{1128F}-\\u{1129D}\\u{1129F}-\\u{112A8}\\u{112B0}-\\u{112DE}\\u{11305}-\\u{1130C}\\u{1130F}-\\u{11310}\\u{11313}-\\u{11328}\\u{1132A}-\\u{11330}\\u{11332}-\\u{11333}\\u{11335}-\\u{11339}\\u{1133D}\\u{11350}\\u{1135D}-\\u{11361}\\u{11380}-\\u{11389}\\u{1138B}\\u{1138E}\\u{11390}-\\u{113B5}\\u{113B7}\\u{113D1}\\u{113D3}\\u{11400}-\\u{11434}\\u{11447}-\\u{1144A}\\u{1145F}-\\u{11461}\\u{11480}-\\u{114AF}\\u{114C4}-\\u{114C5}\\u{114C7}\\u{11580}-\\u{115AE}\\u{115D8}-\\u{115DB}\\u{11600}-\\u{1162F}\\u{11644}\\u{11680}-\\u{116AA}\\u{116B8}\\u{11700}-\\u{1171A}\\u{11740}-\\u{11746}\\u{11800}-\\u{1182B}\\u{118A0}-\\u{118DF}\\u{118FF}-\\u{11906}\\u{11909}\\u{1190C}-\\u{11913}\\u{11915}-\\u{11916}\\u{11918}-\\u{1192F}\\u{1193F}\\u{11941}\\u{119A0}-\\u{119A7}\\u{119AA}-\\u{119D0}\\u{119E1}\\u{119E3}\\u{11A00}\\u{11A0B}-\\u{11A32}\\u{11A3A}\\u{11A50}\\u{11A5C}-\\u{11A89}\\u{11A9D}\\u{11AB0}-\\u{11AF8}\\u{11BC0}-\\u{11BE0}\\u{11C00}-\\u{11C08}\\u{11C0A}-\\u{11C2E}\\u{11C40}\\u{11C72}-\\u{11C8F}\\u{11D00}-\\u{11D06}\\u{11D08}-\\u{11D09}\\u{11D0B}-\\u{11D30}\\u{11D46}\\u{11D60}-\\u{11D65}\\u{11D67}-\\u{11D68}\\u{11D6A}-\\u{11D89}\\u{11D98}\\u{11DB0}-\\u{11DDB}\\u{11EE0}-\\u{11EF2}\\u{11F02}\\u{11F04}-\\u{11F10}\\u{11F12}-\\u{11F33}\\u{11FB0}\\u{12000}-\\u{12399}\\u{12480}-\\u{12543}\\u{12F90}-\\u{12FF0}\\u{13000}-\\u{1342F}\\u{13441}-\\u{13446}\\u{13460}-\\u{143FA}\\u{14400}-\\u{14646}\\u{16100}-\\u{1611D}\\u{16800}-\\u{16A38}\\u{16A40}-\\u{16A5E}\\u{16A70}-\\u{16ABE}\\u{16AD0}-\\u{16AED}\\u{16B00}-\\u{16B2F}\\u{16B40}-\\u{16B43}\\u{16B63}-\\u{16B77}\\u{16B7D}-\\u{16B8F}\\u{16D40}-\\u{16D6C}\\u{16E40}-\\u{16E7F}\\u{16EA0}-\\u{16EB8}\\u{16EBB}-\\u{16ED3}\\u{16F00}-\\u{16F4A}\\u{16F50}\\u{16F93}-\\u{16F9F}\\u{16FE0}-\\u{16FE1}\\u{16FE3}\\u{16FF2}-\\u{16FF3}\\u{17000}-\\u{18CD5}\\u{18CFF}-\\u{18D1E}\\u{18D80}-\\u{18DF2}\\u{1AFF0}-\\u{1AFF3}\\u{1AFF5}-\\u{1AFFB}\\u{1AFFD}-\\u{1AFFE}\\u{1B000}-\\u{1B122}\\u{1B132}\\u{1B150}-\\u{1B152}\\u{1B155}\\u{1B164}-\\u{1B167}\\u{1B170}-\\u{1B2FB}\\u{1BC00}-\\u{1BC6A}\\u{1BC70}-\\u{1BC7C}\\u{1BC80}-\\u{1BC88}\\u{1BC90}-\\u{1BC99}\\u{1D400}-\\u{1D454}\\u{1D456}-\\u{1D49C}\\u{1D49E}-\\u{1D49F}\\u{1D4A2}\\u{1D4A5}-\\u{1D4A6}\\u{1D4A9}-\\u{1D4AC}\\u{1D4AE}-\\u{1D4B9}\\u{1D4BB}\\u{1D4BD}-\\u{1D4C3}\\u{1D4C5}-\\u{1D505}\\u{1D507}-\\u{1D50A}\\u{1D50D}-\\u{1D514}\\u{1D516}-\\u{1D51C}\\u{1D51E}-\\u{1D539}\\u{1D53B}-\\u{1D53E}\\u{1D540}-\\u{1D544}\\u{1D546}\\u{1D54A}-\\u{1D550}\\u{1D552}-\\u{1D6A5}\\u{1D6A8}-\\u{1D6C0}\\u{1D6C2}-\\u{1D6DA}\\u{1D6DC}-\\u{1D6FA}\\u{1D6FC}-\\u{1D714}\\u{1D716}-\\u{1D734}\\u{1D736}-\\u{1D74E}\\u{1D750}-\\u{1D76E}\\u{1D770}-\\u{1D788}\\u{1D78A}-\\u{1D7A8}\\u{1D7AA}-\\u{1D7C2}\\u{1D7C4}-\\u{1D7CB}\\u{1DF00}-\\u{1DF1E}\\u{1DF25}-\\u{1DF2A}\\u{1E030}-\\u{1E06D}\\u{1E100}-\\u{1E12C}\\u{1E137}-\\u{1E13D}\\u{1E14E}\\u{1E290}-\\u{1E2AD}\\u{1E2C0}-\\u{1E2EB}\\u{1E4D0}-\\u{1E4EB}\\u{1E5D0}-\\u{1E5ED}\\u{1E5F0}\\u{1E6C0}-\\u{1E6DE}\\u{1E6E0}-\\u{1E6E2}\\u{1E6E4}-\\u{1E6E5}\\u{1E6E7}-\\u{1E6ED}\\u{1E6F0}-\\u{1E6F4}\\u{1E6FE}-\\u{1E6FF}\\u{1E7E0}-\\u{1E7E6}\\u{1E7E8}-\\u{1E7EB}\\u{1E7ED}-\\u{1E7EE}\\u{1E7F0}-\\u{1E7FE}\\u{1E800}-\\u{1E8C4}\\u{1E900}-\\u{1E943}\\u{1E94B}\\u{1EE00}-\\u{1EE03}\\u{1EE05}-\\u{1EE1F}\\u{1EE21}-\\u{1EE22}\\u{1EE24}\\u{1EE27}\\u{1EE29}-\\u{1EE32}\\u{1EE34}-\\u{1EE37}\\u{1EE39}\\u{1EE3B}\\u{1EE42}\\u{1EE47}\\u{1EE49}\\u{1EE4B}\\u{1EE4D}-\\u{1EE4F}\\u{1EE51}-\\u{1EE52}\\u{1EE54}\\u{1EE57}\\u{1EE59}\\u{1EE5B}\\u{1EE5D}\\u{1EE5F}\\u{1EE61}-\\u{1EE62}\\u{1EE64}\\u{1EE67}-\\u{1EE6A}\\u{1EE6C}-\\u{1EE72}\\u{1EE74}-\\u{1EE77}\\u{1EE79}-\\u{1EE7C}\\u{1EE7E}\\u{1EE80}-\\u{1EE89}\\u{1EE8B}-\\u{1EE9B}\\u{1EEA1}-\\u{1EEA3}\\u{1EEA5}-\\u{1EEA9}\\u{1EEAB}-\\u{1EEBB}\\u{20000}-\\u{2A6DF}\\u{2A700}-\\u{2B81D}\\u{2B820}-\\u{2CEAD}\\u{2CEB0}-\\u{2EBE0}\\u{2EBF0}-\\u{2EE5D}\\u{2F800}-\\u{2FA1D}\\u{30000}-\\u{3134A}\\u{31350}-\\u{33479}\\u0030-\\u0039\\u00B2-\\u00B3\\u00B9\\u00BC-\\u00BE\\u0660-\\u0669\\u06F0-\\u06F9\\u07C0-\\u07C9\\u0966-\\u096F\\u09E6-\\u09EF\\u09F4-\\u09F9\\u0A66-\\u0A6F\\u0AE6-\\u0AEF\\u0B66-\\u0B6F\\u0B72-\\u0B77\\u0BE6-\\u0BF2\\u0C66-\\u0C6F\\u0C78-\\u0C7E\\u0CE6-\\u0CEF\\u0D58-\\u0D5E\\u0D66-\\u0D78\\u0DE6-\\u0DEF\\u0E50-\\u0E59\\u0ED0-\\u0ED9\\u0F20-\\u0F33\\u1040-\\u1049\\u1090-\\u1099\\u1369-\\u137C\\u16EE-\\u16F0\\u17E0-\\u17E9\\u17F0-\\u17F9\\u1810-\\u1819\\u1946-\\u194F\\u19D0-\\u19DA\\u1A80-\\u1A89\\u1A90-\\u1A99\\u1B50-\\u1B59\\u1BB0-\\u1BB9\\u1C40-\\u1C49\\u1C50-\\u1C59\\u2070\\u2074-\\u2079\\u2080-\\u2089\\u2150-\\u2182\\u2185-\\u2189\\u2460-\\u249B\\u24EA-\\u24FF\\u2776-\\u2793\\u2CFD\\u3007\\u3021-\\u3029\\u3038-\\u303A\\u3192-\\u3195\\u3220-\\u3229\\u3248-\\u324F\\u3251-\\u325F\\u3280-\\u3289\\u32B1-\\u32BF\\uA620-\\uA629\\uA6E6-\\uA6EF\\uA830-\\uA835\\uA8D0-\\uA8D9\\uA900-\\uA909\\uA9D0-\\uA9D9\\uA9F0-\\uA9F9\\uAA50-\\uAA59\\uABF0-\\uABF9\\uFF10-\\uFF19\\u{10107}-\\u{10133}\\u{10140}-\\u{10178}\\u{1018A}-\\u{1018B}\\u{102E1}-\\u{102FB}\\u{10320}-\\u{10323}\\u{10341}\\u{1034A}\\u{103D1}-\\u{103D5}\\u{104A0}-\\u{104A9}\\u{10858}-\\u{1085F}\\u{10879}-\\u{1087F}\\u{108A7}-\\u{108AF}\\u{108FB}-\\u{108FF}\\u{10916}-\\u{1091B}\\u{109BC}-\\u{109BD}\\u{109C0}-\\u{109CF}\\u{109D2}-\\u{109FF}\\u{10A40}-\\u{10A48}\\u{10A7D}-\\u{10A7E}\\u{10A9D}-\\u{10A9F}\\u{10AEB}-\\u{10AEF}\\u{10B58}-\\u{10B5F}\\u{10B78}-\\u{10B7F}\\u{10BA9}-\\u{10BAF}\\u{10CFA}-\\u{10CFF}\\u{10D30}-\\u{10D39}\\u{10D40}-\\u{10D49}\\u{10E60}-\\u{10E7E}\\u{10F1D}-\\u{10F26}\\u{10F51}-\\u{10F54}\\u{10FC5}-\\u{10FCB}\\u{11052}-\\u{1106F}\\u{110F0}-\\u{110F9}\\u{11136}-\\u{1113F}\\u{111D0}-\\u{111D9}\\u{111E1}-\\u{111F4}\\u{112F0}-\\u{112F9}\\u{11450}-\\u{11459}\\u{114D0}-\\u{114D9}\\u{11650}-\\u{11659}\\u{116C0}-\\u{116C9}\\u{116D0}-\\u{116E3}\\u{11730}-\\u{1173B}\\u{118E0}-\\u{118F2}\\u{11950}-\\u{11959}\\u{11BF0}-\\u{11BF9}\\u{11C50}-\\u{11C6C}\\u{11D50}-\\u{11D59}\\u{11DA0}-\\u{11DA9}\\u{11DE0}-\\u{11DE9}\\u{11F50}-\\u{11F59}\\u{11FC0}-\\u{11FD4}\\u{12400}-\\u{1246E}\\u{16130}-\\u{16139}\\u{16A60}-\\u{16A69}\\u{16AC0}-\\u{16AC9}\\u{16B50}-\\u{16B59}\\u{16B5B}-\\u{16B61}\\u{16D70}-\\u{16D79}\\u{16E80}-\\u{16E96}\\u{16FF4}-\\u{16FF6}\\u{1CCF0}-\\u{1CCF9}\\u{1D2C0}-\\u{1D2D3}\\u{1D2E0}-\\u{1D2F3}\\u{1D360}-\\u{1D378}\\u{1D7CE}-\\u{1D7FF}\\u{1E140}-\\u{1E149}\\u{1E2F0}-\\u{1E2F9}\\u{1E4F0}-\\u{1E4F9}\\u{1E5F1}-\\u{1E5FA}\\u{1E8C7}-\\u{1E8CF}\\u{1E950}-\\u{1E959}\\u{1EC71}-\\u{1ECAB}\\u{1ECAD}-\\u{1ECAF}\\u{1ECB1}-\\u{1ECB4}\\u{1ED01}-\\u{1ED2D}\\u{1ED2F}-\\u{1ED3D}\\u{1F100}-\\u{1F10C}\\u{1FBF0}-\\u{1FBF9}]+", "u")).filter(Boolean).slice(0, DATASET_FILTER_WORDS_MAX);
+}
+/** A word's prefixes, 1 to {@link DATASET_FILTER_PREFIX_MAX} characters. */ function prefixes(word) {
+    const chars = Array.from(word).slice(0, DATASET_FILTER_PREFIX_MAX);
+    return chars.map((_char, at)=>chars.slice(0, at + 1).join(''));
+}
+/** A word clipped the way its stored prefixes are. */ function clipWord(word) {
+    return Array.from(word).slice(0, DATASET_FILTER_PREFIX_MAX).join('');
+}
+const isEnum = (field)=>{
+    var _ref;
+    var _field_validation_options, _field_validation;
+    return field.type === 'text' && ((_ref = (_field_validation = field.validation) == null ? void 0 : (_field_validation_options = _field_validation.options) == null ? void 0 : _field_validation_options.length) != null ? _ref : 0) > 0;
+};
+const isNumeric = (field)=>field.type === 'int32' || field.type === 'int64' || field.type === 'float';
+/**
+ * A stored value as text, or null. Numbers are accepted because a form or an
+ * automation stores every value as text and an import may do the reverse.
+ */ function asText(value) {
+    if (typeof value === 'string') return value;
+    if (typeof value === 'number' && Number.isFinite(value)) return String(value);
+    return null;
+}
+/**
+ * Plain text as its stored key — its `=` token's value and its
+ * `filterValues` entry: lower-cased, trimmed, clipped to
+ * {@link DATASET_FILTER_VALUE_MAX} characters. What the records table asks a
+ * typed value to equal.
+ */ export function datasetFilterTextKey(value) {
+    return Array.from(value.trim().toLowerCase()).slice(0, DATASET_FILTER_VALUE_MAX).join('');
+}
+const textKey = datasetFilterTextKey;
+/** A stored number (or its text form) as its `=` token's value. */ function numberKey(value) {
+    const number = typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : Number.NaN;
+    return Number.isFinite(number) ? String(number) : null;
+}
+/** A stored boolean (or its text form) as its `=` token's value. */ function boolKey(value) {
+    if (typeof value === 'boolean') return String(value);
+    if (value === 'true' || value === 'false') return value;
+    return null;
+}
+/** Every field a model declares, in display order and then the rest. */ function modelFieldIds(model) {
+    var _model_order, _model_fields;
+    return [
+        ...new Set([
+            ...(_model_order = model.order) != null ? _model_order : [],
+            ...Object.keys((_model_fields = model.fields) != null ? _model_fields : {})
+        ])
+    ];
+}
+/** The tokens described in {@link DATASET_FILTER_KEYS_MAX}'s comment. */ export function datasetFilterKeys(model, values) {
+    const exact = [];
+    const fieldWords = [];
+    const search = [];
+    const searchable = (text)=>{
+        for (const word of datasetFilterWords(text)){
+            for (const prefix of prefixes(word))search.push(`s:${prefix}`);
+        }
+    };
+    for (const fieldId of modelFieldIds(model)){
+        var _model_fields;
+        const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[fieldId];
+        const stored = values == null ? void 0 : values[fieldId];
+        if (!field || stored == null) continue;
+        if (field.type === 'text') {
+            const text = asText(stored);
+            if (text == null || !text.trim()) continue;
+            if (isEnum(field)) {
+                exact.push(`f:${fieldId}=${text}`);
+            } else {
+                exact.push(`f:${fieldId}=${textKey(text)}`);
+                for (const word of datasetFilterWords(text)){
+                    for (const prefix of prefixes(word)){
+                        fieldWords.push(`f:${fieldId}^${prefix}`);
+                    }
+                }
+            }
+            searchable(text);
+        } else if (field.type === 'bool') {
+            const key = boolKey(stored);
+            if (key) exact.push(`f:${fieldId}=${key}`);
+        } else if (isNumeric(field)) {
+            const key = numberKey(stored);
+            if (key) exact.push(`f:${fieldId}=${key}`);
+        } else if (field.type === 'sorted') {
+            for (const member of Array.isArray(stored) ? stored : [
+                stored
+            ]){
+                var _asText;
+                const text = (_asText = asText(member)) == null ? void 0 : _asText.trim();
+                if (!text) continue;
+                exact.push(`f:${fieldId}=${text}`);
+                searchable(text);
+            }
+        }
+    }
+    const kept = [
+        ...new Set([
+            ...exact,
+            ...fieldWords,
+            ...search
+        ])
+    ].slice(0, DATASET_FILTER_KEYS_MAX);
+    return kept.sort();
+}
+/**
+ * Field ids a query can name as `filterValues.<id>`. A path string splits on
+ * `.`, and the SDKs refuse `~ * / [ ]` in one; the model editor allows only
+ * letters, digits and underscores, but a v1 dataset's column names were
+ * never checked. Backticks are refused too, as the escape a path would need.
+ */ const FILTER_VALUE_ID = /^[^.~*/[\]`]+$/;
+/**
+ * The query path of a field's `filterValues` entry, or null for a field id
+ * no path can name — such a field is stamped no entry, and the records table
+ * offers it no equality (its `contains`, served by `filterKeys`, stands).
+ */ export function datasetFilterValuePath(fieldId) {
+    return FILTER_VALUE_ID.test(fieldId) && !/^__.*__$/.test(fieldId) ? `filterValues.${fieldId}` : null;
+}
+/**
+ * A dataset record's FILTER VALUES: `{ [fieldId]: value }`, one scalar per
+ * field an equality can be asked of, so the records table puts every
+ * equality on its query (AGL-3321).
+ *
+ *   enum text (`validation.options`)  the value exactly, as its `=` token;
+ *   plain text                        {@link datasetFilterTextKey};
+ *   bool                              a boolean;
+ *   int32 / int64 / float             a number.
+ *
+ * A form or an automation stores every value as text, so `'19.50'` is the
+ * number 19.5 and `'true'` the boolean, as in {@link datasetFilterKeys}.
+ * Fields with no value, and timestamps, lists, references, maps, bytes,
+ * coordinates and nil fields, carry no entry: a list is asked by member
+ * (`f:<id>=<member>` in `filterKeys`), and the rest are not asked about by
+ * value.
+ *
+ * ## What it costs
+ *
+ * Unlike `values`, this map is NOT exempt from indexing — that is its whole
+ * point. Each entry is a scalar and costs two automatic index entries
+ * (ascending, descending), so a record costs at most two per model field —
+ * bounded by the model's field count (the `/v1` API caps a model at 100
+ * fields), far inside Firestore's 40,000 entries per document, where
+ * `values` was unbounded because a `map` or `sorted` value may hold anything.
+ */ export function datasetFilterValues(model, values) {
+    const out = {};
+    for (const fieldId of modelFieldIds(model)){
+        var _model_fields;
+        const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[fieldId];
+        const stored = values == null ? void 0 : values[fieldId];
+        if (!field || stored == null || !datasetFilterValuePath(fieldId)) continue;
+        if (field.type === 'text') {
+            const text = asText(stored);
+            if (text == null || !text.trim()) continue;
+            out[fieldId] = isEnum(field) ? text : textKey(text);
+        } else if (field.type === 'bool') {
+            const key = boolKey(stored);
+            if (key) out[fieldId] = key === 'true';
+        } else if (isNumeric(field)) {
+            const key = numberKey(stored);
+            if (key) out[fieldId] = Number(key);
+        }
+    }
+    return out;
+}
+/**
+ * The `filterKeys` token a word-level clause asks for, or null when it is
+ * not one:
+ *
+ *  - plain text `contains` → the `^` token of the value's FIRST word, so a
+ *    `contains` is a word-prefix match, not a mid-string one, and one word
+ *    of several is asked (the records table says so);
+ *  - list `contains` (and `equals`) → the member's `=` token;
+ *  - enum, boolean, number and plain text `equals` → the `=` token of the
+ *    value. The records table asks these of `filterValues` instead, which
+ *    composes with a `contains`; the token remains what the quick search and
+ *    older readers match.
+ *  - everything else — ranges, dates, `isEmpty`, `isAnyOf`, negations — null.
+ *
+ * `clause.field` is the field id. The operator names are the grid's: a
+ * select's `equals`, a number's `=`, a boolean's `is`, a list's `contains`.
+ */ export function datasetFilterToken(model, clause) {
+    var _clause_value;
+    var _model_fields;
+    const field = (_model_fields = model.fields) == null ? void 0 : _model_fields[clause.field];
+    if (!field) return null;
+    const { op } = clause;
+    const value = String((_clause_value = clause.value) != null ? _clause_value : '');
+    if (!value.trim()) return null;
+    const equals = op === 'equals' || op === '=' || op === 'is';
+    if (field.type === 'text') {
+        if (isEnum(field)) return equals ? `f:${clause.field}=${value}` : null;
+        if (equals) return `f:${clause.field}=${textKey(value)}`;
+        if (op === 'contains') {
+            const [first] = datasetFilterWords(value);
+            return first ? `f:${clause.field}^${clipWord(first)}` : null;
+        }
+        return null;
+    }
+    if (field.type === 'bool') {
+        const key = boolKey(value.trim());
+        return equals && key ? `f:${clause.field}=${key}` : null;
+    }
+    if (isNumeric(field)) {
+        const key = numberKey(value);
+        return equals && key ? `f:${clause.field}=${key}` : null;
+    }
+    if (field.type === 'sorted') {
+        return equals || op === 'contains' ? `f:${clause.field}=${value.trim()}` : null;
+    }
+    return null;
+}
+/**
+ * The quick-search token for one typed word, or null for a word with no
+ * letters or digits. A word with punctuation inside it is asked by its first
+ * run — split the search text with {@link datasetFilterWords} to ask for
+ * every run.
+ */ export function datasetSearchToken(word) {
+    const [first] = datasetFilterWords(word);
+    return first ? `s:${clipWord(first)}` : null;
+}
+
+//# sourceMappingURL=dataset-models.js.map