@industream/flowmaker-flowbox-ui-components 1.0.6-poc.1 → 1.1.1

package/dist/DCCatalogEntryPicker/DCCatalogEntryPicker.svelte

@@ -2,37 +2,46 @@
     /**
      * DCCatalogEntryPicker — DataCatalog entry picker with configurable columns.
      *
-     * Composes TagBrowser (searchable table with multi-select) and SelectedTags
-     * (collapsible summary with remove). Owns the DataCatalog fetch lifecycle.
+     * Composes TagBrowser (searchable, paginated table with multi-select) and
+     * SelectedTags (collapsible summary with remove). Owns the DataCatalog fetch
+     * lifecycle, pagination state, and the selected-entry metadata cache.
      *
-     * Uses `client.catalogEntries.search()` (v1.4.0+) for both initial load and
-     * server-side search. Filters by `sourceConnections` to scope entries.
-     *
-     * Load strategy:
-     *   1. Initial load fetches up to MAX_ENTRIES + 1 items.
-     *   2. If ≤ MAX_ENTRIES → client mode (client-side search in TagBrowser).
-     *   3. If > MAX_ENTRIES → server mode (TagBrowser search triggers server search
-     *      with 1s debounce, AbortSignal cancels in-flight requests).
+     * Strategy (see docs/superpowers/specs/2026-06-30-tag-picker-lazy-pagination-design.md):
+     *   - Always load page 1 on open — never a blank table.
+     *   - Lazy "load more" pagination via offset; totalCount drives the counter and end.
+     *   - Search: client-side instant when the whole catalog is loaded (small catalogs),
+     *     debounced server search otherwise.
+     *   - Selected entries are cached independently of the loaded window so their
+     *     metadata always renders.
      *
      * @prop dcApiUrl           — DataCatalog API base URL
      * @prop sourceConnectionId — Filters catalog entries by source connection
      * @prop selectedIds        — (bindable) Array of selected CatalogEntry IDs
      * @prop entries            — (bindable) Loaded catalog entries (exposed for parent access)
-     * @prop columns            — Column keys for the browse table (default: name, dataType, labels)
-     * @prop selectedColumnsDisplay — Column keys for the selected tags panel (default: name, dataType, labels)
+     * @prop columns            — Column keys for the browse table
+     * @prop selectedColumnsDisplay — Column keys for the selected tags panel
      * @prop emptyMessage       — Message when no entries match the source connection
      * @prop onSelectionChange  — Called when selection changes (receives full selectedIds array)
      * @prop onRemove           — Called when a single entry is removed from the selection
      */
-    import { untrack } from 'svelte';
     import { DataCatalogClient } from '@industream/datacatalog-client';
     import type { CatalogEntry } from '@industream/datacatalog-client/dto';
     import { DEFAULT_COLUMNS, DEFAULT_SELECTED_COLUMNS_DISPLAY } from './picker-column-helpers';
+    import {
+        PAGE_SIZE,
+        MATCH_ALL,
+        mergePage,
+        deduplicateEntries,
+        normalizeQuery,
+        isAllLoaded,
+        hasMore,
+        remainingCount,
+        upsertSelectedCache,
+        resolveSelected
+    } from './picker-pagination';
     import TagBrowser from './TagBrowser.svelte';
     import SelectedTags from './SelectedTags.svelte';
 
-    const MAX_ENTRIES = 100;
-
     interface Props {
         dcApiUrl?: string;
         sourceConnectionId?: string;
@@ -58,33 +67,46 @@
     }: Props = $props();
 
     let loading = $state(false);
+    let loadingMore = $state(false);
     let searching = $state(false);
     let error = $state('');
-    let tooMany = $state(false);
+    let loadMoreError = $state('');
+    let totalCount = $state(0);
+    let currentQuery = $state(MATCH_ALL);
+    let offset = $state(0);
 
-    /** Abort controller for cancelling in-flight search requests */
-    let searchAbort: AbortController | null = null;
-
-    /** Cached client instance — recreated when dcApiUrl changes */
-    let client: DataCatalogClient | null = null;
-    let clientUrl = '';
+    /**
+     * Metadata for selected tags, kept across reloads/searches so SelectedTags always
+     * renders. Invariant: every mutation of this Map is paired with an `entries`
+     * reassignment, so the `selectedEntries` derived (which reads both) re-runs and picks
+     * up the change. Do NOT upsert the cache without also reassigning `entries`.
+     */
+    let selectedCache = new Map<string, CatalogEntry>();
 
     /**
-     * Deduplicate entries by id — keep only the first occurrence.
+     * Best-effort total for the current query. The server's search response MAY omit
+     * `totalCount`; when it does, we must not silently cap pagination at one page.
+     * Fall back to: a full page → "at least one more page" (loaded + PAGE_SIZE) so the
+     * load-more affordance stays available; a short page → exactly what is loaded (the end).
      */
-    function deduplicateEntries(items: CatalogEntry[]): CatalogEntry[] {
-        const seen = new Set<string>();
-        const result: CatalogEntry[] = [];
-        for (const item of items) {
-            const id = item.id;
-            if (!seen.has(id)) {
-                seen.add(id);
-                result.push(item);
-            }
-        }
-        return result;
+    function estimateTotal(reported: number | undefined, loadedCount: number, lastPageLen: number): number {
+        if (reported != null) return reported;
+        return lastPageLen < PAGE_SIZE ? loadedCount : loadedCount + PAGE_SIZE;
     }
 
+    /** Abort controller for cancelling in-flight requests. */
+    let fetchAbort: AbortController | null = null;
+
+    /** Cached client instance — recreated when dcApiUrl changes. */
+    let client: DataCatalogClient | null = null;
+    let clientUrl = '';
+
+    /** Whether search runs server-side (large catalog) or client-side (everything loaded). */
+    const serverSearch = $derived(!isAllLoaded(MATCH_ALL, entries.length, totalCount) || currentQuery !== MATCH_ALL);
+
+    /** Resolved entries for the selected-tags panel: window first, cache fallback. */
+    const selectedEntries = $derived(resolveSelected(entries, selectedCache, selectedIds));
+
     function getClient(apiUrl: string): DataCatalogClient {
         if (client && clientUrl === apiUrl) return client;
         client = new DataCatalogClient({ baseUrl: apiUrl });
@@ -96,135 +118,124 @@
         if (dcApiUrl) {
             loadInitial(dcApiUrl, sourceConnectionId);
         } else {
+            fetchAbort?.abort();
             entries = [];
+            totalCount = 0;
             error = '';
-            tooMany = false;
         }
     });
 
-    /**
-     * Initial load — fetch MAX_ENTRIES + 1 to detect overflow.
-     * If ≤ MAX_ENTRIES items, stay in client mode.
-     * If more, switch to server search mode.
-     */
-    async function loadInitial(apiUrl: string, connId?: string) {
-        // Cancel any in-flight search
-        searchAbort?.abort();
-        searchAbort = null;
+    /** (Re)load page 1 for the match-all query. Never leaves the table blank on success. */
+    async function loadInitial(apiUrl: string, connId: string = '') {
+        fetchAbort?.abort();
+        const abort = new AbortController();
+        fetchAbort = abort;
 
         loading = true;
         error = '';
-        tooMany = false;
+        loadMoreError = '';
+        currentQuery = MATCH_ALL;
+        offset = 0;
         entries = [];
+        totalCount = 0;
 
         try {
             const dc = getClient(apiUrl);
-            const result = await dc.catalogEntries.search('%', {
+            const result = await dc.catalogEntries.search(MATCH_ALL, {
                 sourceConnectionIds: connId ? [connId] : [],
-                limit: MAX_ENTRIES + 1
-            });
-            const items = result.items ?? [];
-            const uniqueItems = deduplicateEntries(items);
-
-            if (uniqueItems.length > MAX_ENTRIES) {
-                // Too many for client-side filtering → server-search mode. Still show the
-                // first MAX_ENTRIES as a browsable PREVIEW (with a "not all displayed"
-                // warning in TagBrowser) instead of a blank "type to search" screen.
-                tooMany = true;
-                entries = uniqueItems.slice(0, MAX_ENTRIES);
-            } else {
-                tooMany = false;
-                entries = uniqueItems;
-            }
+                limit: PAGE_SIZE,
+                offset: 0
+            }, abort.signal);
+            if (abort.signal.aborted) return;
+            const items = deduplicateEntries(result.items ?? []);
+            entries = items;
+            totalCount = estimateTotal(result.totalCount, items.length, items.length);
+            offset = items.length;
+            upsertSelectedCache(selectedCache, items);
         } catch (err: any) {
-            error = err.message ?? 'Failed to load catalog entries';
+            if (abort.signal.aborted) return;
+            error = err?.message ?? 'Failed to load catalog entries';
             entries = [];
         } finally {
-            loading = false;
+            if (!abort.signal.aborted) loading = false;
         }
     }
 
-    /**
-     * Server-side search — called by TagBrowser when in server mode (tooMany).
-     * Cancels previous in-flight request via AbortSignal.
-     */
-    function handleServerSearch(query: string) {
+    /** Server-side search (large catalogs). Resets pagination for the new query. */
+    function handleServerSearch(rawQuery: string) {
         if (!dcApiUrl) return;
-
-        // Cancel previous in-flight search
-        searchAbort?.abort();
-
-        if (!query) {
-            // Empty query in too-many mode → restore the first-N preview.
-            searching = false;
-            loadInitial(dcApiUrl, sourceConnectionId);
-            return;
-        }
-
+        fetchAbort?.abort();
         const abort = new AbortController();
-        searchAbort = abort;
+        fetchAbort = abort;
+
+        const query = normalizeQuery(rawQuery);
+        currentQuery = query;
+        offset = 0;
+        entries = [];
+        totalCount = 0;
+        error = '';
+        loadMoreError = '';
         searching = true;
 
         const dc = getClient(dcApiUrl);
         dc.catalogEntries.search(query, {
             sourceConnectionIds: sourceConnectionId ? [sourceConnectionId] : [],
-            limit: MAX_ENTRIES
+            limit: PAGE_SIZE,
+            offset: 0
         }, abort.signal)
             .then((result) => {
                 if (abort.signal.aborted) return;
-                const items = result.items ?? [];
-                entries = deduplicateEntries(items);
+                const items = deduplicateEntries(result.items ?? []);
+                entries = items;
+                totalCount = estimateTotal(result.totalCount, items.length, items.length);
+                offset = items.length;
+                upsertSelectedCache(selectedCache, items);
                 searching = false;
             })
-            .catch((err) => {
+            .catch((err: any) => {
                 if (abort.signal.aborted) return;
-                error = err.message ?? 'Search failed';
+                error = err?.message ?? 'Search failed';
                 entries = [];
                 searching = false;
             });
     }
 
-    /**
-     * Full details of the SELECTED entries, fetched by ID — decoupled from the browse
-     * list (`entries`), which is empty in `tooMany` mode and only holds the current
-     * page/search otherwise. Without this, the SelectedTags panel can only render
-     * selected tags that happen to be in `entries` → on large sources it shows
-     * "No tags selected" / loses the selection visually despite a correct count.
-     */
-    let selectedEntries = $state<CatalogEntry[]>([]);
+    /** Append the next page for the current query. No-op when everything is loaded. */
+    function handleLoadMore() {
+        if (!dcApiUrl || loadingMore || loading || searching) return;
+        if (!hasMore(entries.length, totalCount)) return;
 
-    $effect(() => {
-        const ids = [...selectedIds];   // reactive dependency
-        const url = dcApiUrl;           // reactive dependency
-
-        if (!url || ids.length === 0) {
-            selectedEntries = [];
-            return;
-        }
-
-        // Reuse already-resolved entries that are still selected; fetch only the missing.
-        const idSet = new Set(ids);
-        const current = untrack(() => selectedEntries);
-        const keep = current.filter((e) => idSet.has(e.id));
-        const have = new Set(keep.map((e) => e.id));
-        const missing = ids.filter((id) => !have.has(id));
+        fetchAbort?.abort();
+        const abort = new AbortController();
+        fetchAbort = abort;
 
-        if (missing.length === 0) {
-            if (keep.length !== current.length) selectedEntries = keep;
-            return;
-        }
+        loadingMore = true;
+        loadMoreError = '';
+        const startOffset = offset;
 
-        // get() auto-switches to POST /catalog-entries/query for large id lists (URI-safe).
-        getClient(url).catalogEntries
-            .get({ ids: missing })
-            .then((res) => {
-                const latest = new Set(untrack(() => selectedIds));
-                selectedEntries = [...keep, ...(res.items ?? [])].filter((e) => latest.has(e.id));
+        const dc = getClient(dcApiUrl);
+        dc.catalogEntries.search(currentQuery, {
+            sourceConnectionIds: sourceConnectionId ? [sourceConnectionId] : [],
+            limit: PAGE_SIZE,
+            offset: startOffset
+        }, abort.signal)
+            .then((result) => {
+                if (abort.signal.aborted) return;
+                const items = result.items ?? [];
+                entries = mergePage(entries, items);
+                totalCount = estimateTotal(result.totalCount, entries.length, items.length);
+                offset = entries.length;
+                upsertSelectedCache(selectedCache, items);
+                loadingMore = false;
             })
-            .catch(() => {
-                // Keep what we have; SelectedTags falls back to the browse list below.
+            .catch((err: any) => {
+                if (abort.signal.aborted) return;
+                loadMoreError = err?.message ?? 'Failed to load more entries';
+                loadingMore = false;
             });
-    });
+    }
+
+    const remaining = $derived(remainingCount(entries.length, totalCount));
 </script>
 
 <TagBrowser
@@ -232,17 +243,22 @@
     bind:selectedIds
     {columns}
     {loading}
+    {loadingMore}
     {error}
+    {loadMoreError}
     {emptyMessage}
-    {onSelectionChange}
-    onSearch={tooMany ? handleServerSearch : null}
+    {totalCount}
+    {remaining}
+    serverSearch={serverSearch}
     {searching}
-    {tooMany}
+    {onSelectionChange}
+    onSearch={handleServerSearch}
+    onLoadMore={handleLoadMore}
 />
 
 {#if selectedIds.length > 0}
     <SelectedTags
-        entries={selectedEntries.length > 0 ? selectedEntries : entries}
+        entries={selectedEntries}
         bind:selectedIds
         columns={selectedColumnsDisplay}
         {onRemove}

package/dist/DCCatalogEntryPicker/TagBrowser.svelte

@@ -1,4 +1,5 @@
 <script>
+    import { onDestroy } from 'svelte';
     import { DEFAULT_COLUMNS, resolveColumn, resolveValue, resolveSearchText } from './picker-column-helpers';
 
     let {
@@ -6,18 +7,23 @@
         selectedIds = $bindable([]),
         columns = DEFAULT_COLUMNS,
         loading = false,
+        loadingMore = false,
         error = '',
+        loadMoreError = '',
         emptyMessage = 'No entries found for this source connection.',
+        totalCount = 0,
+        remaining = 0,
+        serverSearch = false,
+        searching = false,
         onSelectionChange = null,
         onSearch = null,
-        searching = false,
-        tooMany = false
+        onLoadMore = null
     } = $props();
 
     let searchQuery = $state('');
     let debounceTimer = null;
 
-    /** Resolved column descriptors for rendering — maps each column key to its label, resolver and render type */
+    /** Resolved column descriptors for rendering. */
     const columnDescriptors = $derived(
         columns
             .map(k => { const col = resolveColumn(k); return col ? { key: k, ...col } : null; })
@@ -25,31 +31,40 @@
     );
 
     /**
-     * When onSearch is set (server mode), entries are already filtered server-side.
-     * When onSearch is NOT set (client mode), filter locally.
+     * In server-search mode entries are already filtered server-side.
+     * Otherwise (whole catalog loaded) filter the in-memory entries locally.
      */
     const filtered = $derived(() => {
-        if (onSearch) return entries;
+        if (serverSearch) return entries;
         if (!searchQuery.trim()) return entries;
         const q = searchQuery.toLowerCase();
         return entries.filter(e => resolveSearchText(e, columns).includes(q));
     });
 
     const selectedCount = $derived(selectedIds.length);
-    const totalCount = $derived(entries.length);
     const allVisibleSelected = $derived(filtered().length > 0 && filtered().every(e => selectedIds.includes(e.id)));
 
+    /** Whether the load-more affordance should show (more pages for the current query). */
+    const showLoadMore = $derived(remaining > 0 && !loading && !error);
+
     function handleSearchInput(e) {
         searchQuery = e.target.value;
-        if (!onSearch) return;
 
-        // Debounce 1s for server search
+        // Always cancel a pending server search: if the mode is (or just became)
+        // client-side, no request should fire; if server-side, we re-arm below.
         if (debounceTimer) clearTimeout(debounceTimer);
+        if (!serverSearch) return; // client-side filtering is reactive via filtered()
+
+        // Debounce 1s for server search
         debounceTimer = setTimeout(() => {
-            onSearch(searchQuery.trim());
+            onSearch?.(searchQuery.trim());
         }, 1000);
     }
 
+    onDestroy(() => {
+        if (debounceTimer) clearTimeout(debounceTimer);
+    });
+
     function toggleEntry(id) {
         selectedIds = selectedIds.indexOf(id) >= 0
             ? selectedIds.filter(s => s !== id)
@@ -73,6 +88,21 @@
     function toggleAll() {
         allVisibleSelected ? deselectAll() : selectAll();
     }
+
+    /**
+     * Svelte action: fires `onLoadMore` when the sentinel scrolls into the
+     * viewport of the scrollable table container. The "Load more" button is
+     * the accessible/explicit fallback for the same callback.
+     */
+    function autoLoad(node) {
+        const observer = new IntersectionObserver((entriesObserved) => {
+            if (entriesObserved.some(en => en.isIntersecting) && !loadingMore) {
+                onLoadMore?.();
+            }
+        }, { root: node.closest('.table-container'), rootMargin: '120px' });
+        observer.observe(node);
+        return { destroy() { observer.disconnect(); } };
+    }
 </script>
 
 <div class="tag-browser">
@@ -84,7 +114,7 @@
             <input
                 type="text"
                 class="search-input"
-                placeholder={tooMany ? "Search to find entries..." : "Search..."}
+                placeholder={serverSearch ? "Search to find entries..." : "Search..."}
                 value={searchQuery}
                 oninput={handleSearchInput}
             />
@@ -107,14 +137,8 @@
             <span>{error}</span>
         </div>
     {:else if entries.length === 0 && !searching}
-        <div class="state-message empty-state"><span>{tooMany ? 'No matching entries. Refine your search.' : emptyMessage}</span></div>
+        <div class="state-message empty-state"><span>{emptyMessage}</span></div>
     {:else}
-        {#if tooMany}
-            <div class="too-many-banner">
-                <svg viewBox="0 0 16 16" width="16" height="16"><path d="M8 1a7 7 0 1 0 0 14A7 7 0 0 0 8 1zm-.75 3.5h1.5v5h-1.5zm.75 8a1 1 0 1 1 0-2 1 1 0 0 1 0 2z" fill="currentColor"></path></svg>
-                <span>Showing {entries.length} entries (not all). Use the search box to find others.</span>
-            </div>
-        {/if}
         <div class="table-container">
             <table class="entry-table">
                 <!-- Table headers: one checkbox column + one column per columnDescriptor -->
@@ -156,8 +180,22 @@
                     {/each}
                 </tbody>
             </table>
+            {#if showLoadMore}
+                <div class="load-more" use:autoLoad>
+                    {#if loadingMore}
+                        <div class="spinner"></div> <span>Loading more…</span>
+                    {:else}
+                        <button class="action-button" onclick={() => onLoadMore?.()}>
+                            Load more ({remaining} remaining)
+                        </button>
+                    {/if}
+                </div>
+                {#if loadMoreError}
+                    <div class="load-more-error">{loadMoreError} — <button class="link-button" onclick={() => onLoadMore?.()}>retry</button></div>
+                {/if}
+            {/if}
         </div>
-        {#if !onSearch && filtered().length !== entries.length}
+        {#if !serverSearch && filtered().length !== entries.length}
             <div class="filter-info">Showing {filtered().length} of {entries.length} entries</div>
         {/if}
     {/if}
@@ -201,14 +239,6 @@
     }
     .error-state { color: var(--cds-support-error, #fa4d56); }
     .empty-state { color: var(--cds-text-helper, #8d8d8d); }
-    .too-many-banner {
-        display: flex; align-items: center; gap: 0.5rem;
-        padding: 0.5rem 0.75rem;
-        background: var(--cds-notification-background-warning, #3d3522);
-        border-left: 3px solid var(--cds-support-warning, #f1c21b);
-        color: var(--cds-text-primary, #f4f4f4); font-size: 0.75rem;
-    }
-    .too-many-banner svg { color: var(--cds-support-warning, #f1c21b); flex: 0 0 auto; }
     .spinner {
         width: 1rem; height: 1rem;
         border: 2px solid var(--cds-border-subtle-01, #e0e0e0);
@@ -260,4 +290,18 @@
         font-size: 0.75rem; color: var(--cds-text-helper, #8d8d8d);
         text-align: center; padding: 0.25rem;
     }
+    .load-more {
+        display: flex; align-items: center; justify-content: center;
+        gap: 0.5rem; padding: 0.75rem; font-size: 0.8125rem;
+        color: var(--cds-text-secondary, #c6c6c6);
+    }
+    .load-more-error {
+        font-size: 0.75rem; color: var(--cds-support-error, #fa4d56);
+        text-align: center; padding: 0.25rem;
+    }
+    .link-button {
+        background: none; border: none; padding: 0; cursor: pointer;
+        color: var(--cds-link-primary, #78a9ff); text-decoration: underline;
+        font-size: inherit;
+    }
 </style>

package/dist/DCCatalogEntryPicker/TagBrowser.svelte.d.ts

@@ -8,23 +8,33 @@ declare const TagBrowser: import("svelte").Component<{
     selectedIds?: any[];
     columns?: typeof DEFAULT_COLUMNS;
     loading?: boolean;
+    loadingMore?: boolean;
     error?: string;
+    loadMoreError?: string;
     emptyMessage?: string;
+    totalCount?: number;
+    remaining?: number;
+    serverSearch?: boolean;
+    searching?: boolean;
     onSelectionChange?: any;
     onSearch?: any;
-    searching?: boolean;
-    tooMany?: boolean;
+    onLoadMore?: any;
 }, {}, "selectedIds">;
 type $$ComponentProps = {
     entries?: any[];
     selectedIds?: any[];
     columns?: typeof DEFAULT_COLUMNS;
     loading?: boolean;
+    loadingMore?: boolean;
     error?: string;
+    loadMoreError?: string;
     emptyMessage?: string;
+    totalCount?: number;
+    remaining?: number;
+    serverSearch?: boolean;
+    searching?: boolean;
     onSelectionChange?: any;
     onSearch?: any;
-    searching?: boolean;
-    tooMany?: boolean;
+    onLoadMore?: any;
 };
 import { DEFAULT_COLUMNS } from './picker-column-helpers';

package/dist/DCCatalogEntryPicker/picker-pagination.d.ts

@@ -0,0 +1,24 @@
+import type { CatalogEntry } from '@industream/datacatalog-client/dto';
+/** Number of entries fetched per page. */
+export declare const PAGE_SIZE = 100;
+/** Query token that matches all entries (server-side wildcard). */
+export declare const MATCH_ALL = "%";
+/** Deduplicate entries by id, keeping the first occurrence and preserving order. */
+export declare function deduplicateEntries(items: CatalogEntry[]): CatalogEntry[];
+/** Append a freshly fetched page to the accumulated list, dropping duplicates. */
+export declare function mergePage(existing: CatalogEntry[], page: CatalogEntry[]): CatalogEntry[];
+/** Trim a raw search input; an empty query becomes the match-all token. */
+export declare function normalizeQuery(raw: string): string;
+/** True when the full match-all set is in memory (small catalog → client-side search). */
+export declare function isAllLoaded(query: string, loadedCount: number, totalCount: number): boolean;
+/** Whether more pages remain for the current query. */
+export declare function hasMore(loadedCount: number, totalCount: number): boolean;
+/** Number of not-yet-loaded entries for the current query (never negative). */
+export declare function remainingCount(loadedCount: number, totalCount: number): number;
+/** Upsert every fetched entry into the selected-entry metadata cache (mutates and returns it). */
+export declare function upsertSelectedCache(cache: Map<string, CatalogEntry>, items: CatalogEntry[]): Map<string, CatalogEntry>;
+/**
+ * Resolve the selected entries for display: prefer the currently loaded window,
+ * fall back to the cache, in the order of `selectedIds`. Ids absent from both are skipped.
+ */
+export declare function resolveSelected(entries: CatalogEntry[], cache: Map<string, CatalogEntry>, selectedIds: string[]): CatalogEntry[];