@aglyn/plugins-data 1.0.0-beta.198 → 1.0.0-beta.200

package/src/lib/components/host-datasets-card.component.js

@@ -21,6 +21,7 @@ import { exportShortfall, mapImportColumns, parseImportRows } from "../model/ind
 import { CardDisplay, useConfirmationContext } from "@aglyn/shared-ui-jsx";
 import ListFilterChips from "@aglyn/shared-ui-jsx/components/list-filter-chips.component";
 import { ListPagination } from "@aglyn/shared-ui-jsx/components/list-pagination.component";
+import ListQueryNotices, { listQueryRefusals } from "@aglyn/shared-ui-jsx/components/list-query-notices.component";
 import { ListTable, listActionsColumn } from "@aglyn/shared-ui-jsx/components/list-table.component";
 import QuotaReadoutComponent from "@aglyn/shared-ui-jsx/components/quota-readout.component";
 import { listFilterGridColumns } from "@aglyn/shared-ui-jsx/const/list-grid-filter";
@@ -31,10 +32,11 @@ import { Button, Dialog, DialogActions, DialogContent, DialogTitle, MenuItem, St
 import { collection, deleteDoc, deleteField, doc, documentId, getCountFromServer, getDocs, limit, orderBy, query, where, setDoc, writeBatch } from "firebase/firestore";
 import { useCallback, useEffect, useMemo, useState } from "react";
 import { useFirestore, useFirestoreCollection, useHostActivityLogger, useOrgDataScope, usePagedCollection, useScopeTokens, useUser } from "@aglyn/tenant-feature-instance";
+import { listQueryConstraints } from "@aglyn/tenant-feature-instance/hooks/use-list-query";
 import { authorizedFetch } from "@aglyn/shared-util-http/authorized-token";
 import { DatasetSchemaDialog } from "./dataset-schema-dialog.component.js";
 import { DatasetRecordDialog } from "./dataset-record-dialog.component.js";
-import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER_WINDOW, recordColumn } from "./dataset-record-filter.js";
+import { datasetRecordFilter, planRecordQuery, recordColumn } from "./dataset-record-filter.js";
 /**
  * How many existing records a keyed CSV import may consult.
  *
@@ -66,7 +68,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
  * dark-launch rule as everywhere (no plan, no gate). The host path is
  * the pre-migration fallback for hosts not yet org-wired.
  */ export function HostDatasetsCard(props) {
-    var _datasets_find, _ref, _ref1, _ref2, _ref3, _ref4, _ref5, _ref6;
+    var _datasets_find, _ref, _ref1, _ref2, _ref3, _ref4, _ref5;
     const { hostId } = props;
     // `dataScope` is null until the async org lookup settles (AGL-1061), and
     // for a host with no owning org — there is no host fallback to act on
@@ -264,24 +266,23 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
    * rows would run in one order within a page and another across pages.
    * The grid's column sort is off for the same reason.
    *
-   * ## Filtering, and what reaches every record
+   * ## Filtering: every clause and the search word are on the query
    *
-   * The grid's Filters panel and quick search narrow the walk through the
-   * record's `filterKeys` (`datasetFilterKeys`): a token array Firestore's
-   * automatic single-field index serves with `array-contains` beside the same
-   * document-name order, so it needs no composite index and no index
-   * deploy. ONE condition is served that way — the first clause a token can
-   * answer (`datasetFilterToken`), else the first search word — and when it is
-   * the only condition, the walk pages through every matching record exactly
-   * as it pages through all of them.
+   * The grid's Filters panel and quick search are predicates on this same
+   * walk (AGL-3321), planned by `planRecordQuery` (`./dataset-record-filter`,
+   * over the shared `planListQuery`): equalities ask the record's
+   * `filterValues`, and one word-level clause — a text `contains`, a list
+   * member, or the search word — asks its `filterKeys`. Firestore's
+   * automatic single-field indexes merge them under the same document-name
+   * order, so every filter pages through every matching record exactly as
+   * the plain walk pages through all of them, with no composite index and
+   * no index deploy.
    *
-   * Anything more — a second clause, a second word, a multi-word `contains`,
-   * or a clause no token answers (number and date ranges, empty checks,
-   * negations) — cannot be one `array-contains`. Those read a WINDOW of
-   * {@link RECORD_FILTER_WINDOW} records under the served condition (or the
-   * plain walk when none is servable), match the rest in memory, and page the
-   * matches here; the caption says when the window was full. Ranges on
-   * numbers and dates are therefore always window-only.
+   * Nothing is matched over the rows already loaded. What one query cannot
+   * hold is not offered (ranges, negations, empty checks) or is refused by
+   * name above the table (a second `contains`, a `contains` beside the
+   * search), and a many-word `contains` or search says it asked the first
+   * word.
    */ const gridFilter = useListGridFilter({
         selectFields: recordFilter.selectFields
     });
@@ -291,62 +292,32 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
         gridFilter.clauses,
         recordFilter.fields
     ]);
-    const recordPlan = useMemo(()=>planRecordFilter(model, recordClauses, gridFilter.searchWords), [
+    const recordPlan = useMemo(()=>planRecordQuery(model, recordFilter, recordClauses, gridFilter.searchWords), [
         model,
+        recordFilter,
         recordClauses,
         gridFilter.searchWords
     ]);
-    /** The walk, narrowed by the served token when there is one. */ const recordsQuery = useCallback((cap)=>dataScope && (selected == null ? void 0 : selected.$id) ? query(collection(firestore, dataScope[0], dataScope[1], 'datasets', selected.$id, 'records'), ...recordPlan.token ? [
-            where('filterKeys', 'array-contains', recordPlan.token)
-        ] : [], orderBy(documentId()), limit(cap)) : null, [
+    // The query's identity: a new plan is a new walk, from page one.
+    const recordPlanKey = JSON.stringify({
+        filters: recordPlan.plan.filters,
+        orderBy: recordPlan.plan.orderBy
+    });
+    const { rows: records, hasMore: hasMoreRecords, page: recordPage, setPage: setRecordPage, pageSize: recordPageSize, setPageSize: setRecordPageSize } = usePagedCollection((pageLimit)=>dataScope && (selected == null ? void 0 : selected.$id) ? query(collection(firestore, dataScope[0], dataScope[1], 'datasets', selected.$id, 'records'), ...listQueryConstraints(recordPlan.plan), limit(pageLimit)) : null, [
         firestore,
         dataScope,
         selected == null ? void 0 : selected.$id,
-        recordPlan.token
-    ]);
-    const { rows: pagedRecords, hasMore: hasMorePaged, page: pagedPage, setPage: setPagedPage, pageSize: recordPageSize, setPageSize: setPagedPageSize } = usePagedCollection((pageLimit)=>recordPlan.windowed ? null : recordsQuery(pageLimit), [
-        recordsQuery,
-        recordPlan.windowed
+        recordPlanKey
     ], {
         idField: '$id'
     });
-    // The window: one read past the cap, so "the window was full" is a fact.
-    const { data: windowDocs } = useFirestoreCollection(()=>recordPlan.windowed ? recordsQuery(RECORD_FILTER_WINDOW + 1) : null, [
-        recordsQuery,
-        recordPlan.windowed
-    ], {
-        idField: '$id'
-    });
-    const windowRecords = useMemo(()=>recordPlan.windowed ? (windowDocs != null ? windowDocs : []).slice(0, RECORD_FILTER_WINDOW) : [], [
-        recordPlan.windowed,
-        windowDocs
-    ]);
-    const windowFull = recordPlan.windowed && ((_ref = windowDocs == null ? void 0 : windowDocs.length) != null ? _ref : 0) > RECORD_FILTER_WINDOW;
-    const windowMatches = useMemo(()=>recordPlan.windowed ? matchRecordWindow(model, recordFilter.fields, windowRecords, recordPlan) : [], [
-        recordPlan,
-        model,
-        recordFilter.fields,
-        windowRecords
-    ]);
-    const [windowPage, setWindowPage] = useState(0);
-    const [windowPageSize, setWindowPageSize] = useState(recordPageSize);
-    useEffect(()=>{
-        setWindowPage(0);
-    }, [
-        recordPlan,
-        selected == null ? void 0 : selected.$id
-    ]);
-    /** Every record this card has read: the page, or the whole window. */ const records = recordPlan.windowed ? windowRecords : pagedRecords;
-    /** The rows on screen. */ const shownRecords = recordPlan.windowed ? windowMatches.slice(windowPage * windowPageSize, (windowPage + 1) * windowPageSize) : pagedRecords;
-    const recordPage = recordPlan.windowed ? windowPage : pagedPage;
-    const setRecordPage = recordPlan.windowed ? setWindowPage : setPagedPage;
-    const hasMoreRecords = recordPlan.windowed ? (windowPage + 1) * windowPageSize < windowMatches.length : hasMorePaged;
-    const setRecordPageSize = useCallback((next)=>{
-        setPagedPageSize(next);
-        setWindowPageSize(next);
-        setWindowPage(0);
-    }, [
-        setPagedPageSize
+    const recordRefusals = useMemo(()=>listQueryRefusals(recordPlan.refused, {
+            fields: recordFilter.fields,
+            headers: recordFilter.headers,
+            options: recordFilter.options
+        }), [
+        recordPlan.refused,
+        recordFilter
     ]);
     const filteringRecords = recordClauses.length > 0 || gridFilter.searchWords.some((word)=>word.trim());
     /**
@@ -476,7 +447,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
     ]);
     // Entries are human names — "Roast preference" → stable id
     // `roast_preference` with the pretty name kept for headers (AGL-558).
-    const creatorEntries = parseDatasetFieldEntries((_ref1 = creator == null ? void 0 : creator.fields) != null ? _ref1 : '');
+    const creatorEntries = parseDatasetFieldEntries((_ref = creator == null ? void 0 : creator.fields) != null ? _ref : '');
     const creatorFields = creatorEntries.map((entry)=>entry.id);
     const handleCreate = useCallback(async ()=>{
         // No org, no create. The `else` that used to sit below this was the
@@ -816,6 +787,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                         'values',
                         'referencedIds',
                         'filterKeys',
+                        'filterValues',
                         'updatedAt'
                     ]
                 });
@@ -1116,8 +1088,12 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
             const keyWindow = await getDocs(query(collection(firestore, dataScope[0], dataScope[1], 'datasets', selected.$id, 'records'), orderBy(documentId()), limit(IMPORT_KEY_WINDOW))).catch(()=>null);
             for (const snapshot of (_ref1 = keyWindow == null ? void 0 : keyWindow.docs) != null ? _ref1 : []){
                 var _snapshot_get;
-                const key = keyOf((_snapshot_get = snapshot.get('values')) != null ? _snapshot_get : {});
-                if (key) existingByKey.set(key, snapshot.id);
+                const values = (_snapshot_get = snapshot.get('values')) != null ? _snapshot_get : {};
+                const key = keyOf(values);
+                if (key) existingByKey.set(key, {
+                    id: snapshot.id,
+                    values
+                });
             }
             if (recordCount > IMPORT_KEY_WINDOW) {
                 enqueueSnackbar(`Matching on "${keyField}" read the first ` + `${IMPORT_KEY_WINDOW.toLocaleString()} of ${recordCount.toLocaleString()} ` + 'records — rows keyed beyond that will be added rather than updated.', {
@@ -1137,12 +1113,16 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                 ...keyless
             ];
             for (const [key, row] of deduped){
-                const existingId = existingByKey.get(key);
-                if (existingId) updates.push({
-                    id: existingId,
-                    values: row.values
-                });
-                else creates.push(row);
+                const existing = existingByKey.get(key);
+                // The import's columns over the record's own: a column the file
+                // does not carry keeps its value, and the filter fields are derived
+                // from the WHOLE record, not the columns imported.
+                if (existing) {
+                    updates.push({
+                        id: existing.id,
+                        values: _extends({}, existing.values, row.values)
+                    });
+                } else creates.push(row);
             }
         }
         // Against the dataset's real size, not the loaded window (AGL-1716) —
@@ -1167,8 +1147,16 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                     values: update.values
                 }, datasetIntegrityUpdate(model, update.values, deleteField()), {
                     updatedAt: Timestamp.now()
-                }), {
-                    merge: true
+                }), // Each named field replaced whole — `merge: true` would fold the
+                // `filterValues` map into the stored one and keep a cleared value.
+                {
+                    mergeFields: [
+                        'values',
+                        'referencedIds',
+                        'filterKeys',
+                        'filterValues',
+                        'updatedAt'
+                    ]
                 });
             });
             await batch.commit();
@@ -1320,7 +1308,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                                 select: true,
                                 size: "small",
                                 label: "Dataset",
-                                value: (_ref2 = selected == null ? void 0 : selected.$id) != null ? _ref2 : '',
+                                value: (_ref1 = selected == null ? void 0 : selected.$id) != null ? _ref1 : '',
                                 onChange: (event)=>setSelectedId(event.target.value),
                                 sx: {
                                     minWidth: 180
@@ -1386,21 +1374,19 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                                 headers: recordFilter.headers,
                                 clauses: recordClauses,
                                 onChange: gridFilter.setClauses,
-                                options: recordFilter.options,
-                                servedField: recordPlan.windowed ? null : recordPlan.servedField
+                                options: recordFilter.options
+                            }),
+                            /*#__PURE__*/ _jsx(ListQueryNotices, {
+                                refused: recordRefusals,
+                                notices: recordPlan.notices
                             }),
-                            windowFull ? /*#__PURE__*/ _jsx(Typography, {
-                                variant: "caption",
-                                color: "text.secondary",
-                                children: `Filtering the first ${RECORD_FILTER_WINDOW.toLocaleString()} records` + (recordPlan.token ? ' that match the first condition' : '') + ' — one filter or one search word on its own reaches every record.'
-                            }) : null,
                             /*#__PURE__*/ _jsx(ListTable, {
                                 "aria-label": "Records",
-                                rows: shownRecords,
+                                rows: records,
                                 columns: recordGridColumns,
                                 onOpen: (id)=>setViewerId(id),
-                                // One page of the walk (or of the window's matches), turned by
-                                // the footer below: the grid neither slices nor filters it.
+                                // One page of the walk, turned by the footer below: the grid
+                                // neither slices nor filters it.
                                 hideFooter: true,
                                 filterMode: "server",
                                 filterModel: gridFilter.filterModel,
@@ -1412,7 +1398,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                             /*#__PURE__*/ _jsx(ListPagination, {
                                 page: recordPage,
                                 pageSize: recordPageSize,
-                                rowCount: shownRecords.length,
+                                rowCount: records.length,
                                 hasMore: hasMoreRecords,
                                 onPageChange: setRecordPage,
                                 onPageSizeChange: setRecordPageSize
@@ -1464,7 +1450,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                         children: [
                             /*#__PURE__*/ _jsx(TextField, {
                                 label: "Name",
-                                value: (_ref3 = creator == null ? void 0 : creator.name) != null ? _ref3 : '',
+                                value: (_ref2 = creator == null ? void 0 : creator.name) != null ? _ref2 : '',
                                 onChange: (event)=>setCreator((prev)=>prev ? _extends({}, prev, {
                                             name: event.target.value
                                         }) : prev),
@@ -1477,7 +1463,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                             }),
                             /*#__PURE__*/ _jsx(TextField, {
                                 label: "Fields",
-                                value: (_ref4 = creator == null ? void 0 : creator.fields) != null ? _ref4 : '',
+                                value: (_ref3 = creator == null ? void 0 : creator.fields) != null ? _ref3 : '',
                                 onChange: (event)=>setCreator((prev)=>prev ? _extends({}, prev, {
                                             fields: event.target.value
                                         }) : prev),
@@ -1711,7 +1697,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                                 minRows: 6,
                                 size: "small",
                                 label: "CSV (with header row) or JSON array",
-                                value: (_ref5 = importer == null ? void 0 : importer.text) != null ? _ref5 : '',
+                                value: (_ref4 = importer == null ? void 0 : importer.text) != null ? _ref4 : '',
                                 onChange: (event)=>setImporter((prev)=>prev ? _extends({}, prev, {
                                             text: event.target.value
                                         }) : prev),
@@ -1724,7 +1710,7 @@ import { datasetRecordFilter, matchRecordWindow, planRecordFilter, RECORD_FILTER
                                 select: true,
                                 size: "small",
                                 label: "Match on field (upsert)",
-                                value: (_ref6 = importer == null ? void 0 : importer.keyField) != null ? _ref6 : '',
+                                value: (_ref5 = importer == null ? void 0 : importer.keyField) != null ? _ref5 : '',
                                 onChange: (event)=>setImporter((prev)=>prev ? _extends({}, prev, {
                                             keyField: event.target.value
                                         }) : prev),