@industream/flowmaker-flowbox-ui-components 1.0.6-poc.1 → 1.1.1

package/dist/DCCatalogEntryPicker/picker-pagination.js

@@ -0,0 +1,58 @@
+/** Number of entries fetched per page. */
+export const PAGE_SIZE = 100;
+/** Query token that matches all entries (server-side wildcard). */
+export const MATCH_ALL = '%';
+/** Deduplicate entries by id, keeping the first occurrence and preserving order. */
+export function deduplicateEntries(items) {
+    const seen = new Set();
+    const result = [];
+    for (const item of items) {
+        if (!seen.has(item.id)) {
+            seen.add(item.id);
+            result.push(item);
+        }
+    }
+    return result;
+}
+/** Append a freshly fetched page to the accumulated list, dropping duplicates. */
+export function mergePage(existing, page) {
+    return deduplicateEntries([...existing, ...page]);
+}
+/** Trim a raw search input; an empty query becomes the match-all token. */
+export function normalizeQuery(raw) {
+    const trimmed = raw.trim();
+    return trimmed === '' ? MATCH_ALL : trimmed;
+}
+/** True when the full match-all set is in memory (small catalog → client-side search). */
+export function isAllLoaded(query, loadedCount, totalCount) {
+    return query === MATCH_ALL && loadedCount >= totalCount;
+}
+/** Whether more pages remain for the current query. */
+export function hasMore(loadedCount, totalCount) {
+    return loadedCount < totalCount;
+}
+/** Number of not-yet-loaded entries for the current query (never negative). */
+export function remainingCount(loadedCount, totalCount) {
+    return Math.max(0, totalCount - loadedCount);
+}
+/** Upsert every fetched entry into the selected-entry metadata cache (mutates and returns it). */
+export function upsertSelectedCache(cache, items) {
+    for (const item of items) {
+        cache.set(item.id, item);
+    }
+    return cache;
+}
+/**
+ * Resolve the selected entries for display: prefer the currently loaded window,
+ * fall back to the cache, in the order of `selectedIds`. Ids absent from both are skipped.
+ */
+export function resolveSelected(entries, cache, selectedIds) {
+    const windowById = new Map(entries.map(e => [e.id, e]));
+    const result = [];
+    for (const id of selectedIds) {
+        const found = windowById.get(id) ?? cache.get(id);
+        if (found)
+            result.push(found);
+    }
+    return result;
+}

package/dist/DCCatalogEntryPicker/picker-pagination.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/DCCatalogEntryPicker/picker-pagination.test.js

@@ -0,0 +1,61 @@
+import { describe, it, expect } from 'vitest';
+import { PAGE_SIZE, MATCH_ALL, deduplicateEntries, mergePage, normalizeQuery, isAllLoaded, hasMore, remainingCount, upsertSelectedCache, resolveSelected } from './picker-pagination';
+const entry = (id, name = id) => ({ id, name });
+describe('constants', () => {
+    it('exposes the page size and match-all token', () => {
+        expect(PAGE_SIZE).toBe(100);
+        expect(MATCH_ALL).toBe('%');
+    });
+});
+describe('deduplicateEntries', () => {
+    it('keeps first occurrence by id and preserves order', () => {
+        const result = deduplicateEntries([entry('a'), entry('b'), entry('a')]);
+        expect(result.map(e => e.id)).toEqual(['a', 'b']);
+    });
+});
+describe('mergePage', () => {
+    it('appends a new page and drops duplicates against existing', () => {
+        const result = mergePage([entry('a'), entry('b')], [entry('b'), entry('c')]);
+        expect(result.map(e => e.id)).toEqual(['a', 'b', 'c']);
+    });
+});
+describe('normalizeQuery', () => {
+    it('trims and maps empty input to the match-all token', () => {
+        expect(normalizeQuery('   ')).toBe(MATCH_ALL);
+        expect(normalizeQuery('')).toBe(MATCH_ALL);
+    });
+    it('trims a real query', () => {
+        expect(normalizeQuery('  temp ')).toBe('temp');
+    });
+});
+describe('isAllLoaded', () => {
+    it('is true only for match-all with every item loaded', () => {
+        expect(isAllLoaded(MATCH_ALL, 50, 50)).toBe(true);
+        expect(isAllLoaded(MATCH_ALL, 50, 120)).toBe(false);
+        expect(isAllLoaded('temp', 50, 50)).toBe(false);
+    });
+});
+describe('hasMore / remainingCount', () => {
+    it('reports remaining pages relative to total', () => {
+        expect(hasMore(100, 350)).toBe(true);
+        expect(hasMore(350, 350)).toBe(false);
+        expect(remainingCount(100, 350)).toBe(250);
+        expect(remainingCount(350, 350)).toBe(0);
+        expect(remainingCount(400, 350)).toBe(0);
+    });
+});
+describe('selected cache', () => {
+    it('upserts fetched items by id', () => {
+        const cache = upsertSelectedCache(new Map(), [entry('a'), entry('b')]);
+        upsertSelectedCache(cache, [entry('b', 'renamed')]);
+        expect(cache.get('b')?.name).toBe('renamed');
+        expect(cache.size).toBe(2);
+    });
+    it('resolves selected entries from window first, cache as fallback, in selection order', () => {
+        const cache = upsertSelectedCache(new Map(), [entry('x'), entry('y')]);
+        const window = [entry('y', 'in-window')];
+        const result = resolveSelected(window, cache, ['x', 'y']);
+        expect(result.map(e => e.id)).toEqual(['x', 'y']);
+        expect(result.find(e => e.id === 'y')?.name).toBe('in-window');
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@industream/flowmaker-flowbox-ui-components",
-  "version": "1.0.6-poc.1",
+  "version": "1.1.1",
   "description": "Reusable Svelte components for FlowMaker FlowBox UI",
   "type": "module",
   "svelte": "./dist/index.js",
@@ -40,11 +40,12 @@
   ],
   "scripts": {
     "build": "svelte-package --input src",
-    "dev": "vite --config vite.dev.config.ts"
+    "test": "vitest",
+    "test:run": "vitest run"
   },
   "peerDependencies": {
     "svelte": "^5.0.0",
-    "@industream/datacatalog-client": "1.9.2"
+    "@industream/datacatalog-client": "^1.9.1"
   },
   "devDependencies": {
     "@industream/datacatalog-client": "1.9.2",
@@ -52,7 +53,8 @@
     "vite": "^6.0.0",
     "@sveltejs/vite-plugin-svelte": "^5.0.0",
     "@sveltejs/package": "^2.0.0",
-    "typescript": "^5.0.0"
+    "typescript": "^5.0.0",
+    "vitest": "^2.1.0"
   },
   "publishConfig": {
     "access": "public"

package/src/DCCatalogEntryPicker/DCCatalogEntryPicker.svelte

@@ -2,37 +2,46 @@
     /**
      * DCCatalogEntryPicker — DataCatalog entry picker with configurable columns.
      *
-     * Composes TagBrowser (searchable table with multi-select) and SelectedTags
-     * (collapsible summary with remove). Owns the DataCatalog fetch lifecycle.
+     * Composes TagBrowser (searchable, paginated table with multi-select) and
+     * SelectedTags (collapsible summary with remove). Owns the DataCatalog fetch
+     * lifecycle, pagination state, and the selected-entry metadata cache.
      *
-     * Uses `client.catalogEntries.search()` (v1.4.0+) for both initial load and
-     * server-side search. Filters by `sourceConnections` to scope entries.
-     *
-     * Load strategy:
-     *   1. Initial load fetches up to MAX_ENTRIES + 1 items.
-     *   2. If ≤ MAX_ENTRIES → client mode (client-side search in TagBrowser).
-     *   3. If > MAX_ENTRIES → server mode (TagBrowser search triggers server search
-     *      with 1s debounce, AbortSignal cancels in-flight requests).
+     * Strategy (see docs/superpowers/specs/2026-06-30-tag-picker-lazy-pagination-design.md):
+     *   - Always load page 1 on open — never a blank table.
+     *   - Lazy "load more" pagination via offset; totalCount drives the counter and end.
+     *   - Search: client-side instant when the whole catalog is loaded (small catalogs),
+     *     debounced server search otherwise.
+     *   - Selected entries are cached independently of the loaded window so their
+     *     metadata always renders.
      *
      * @prop dcApiUrl           — DataCatalog API base URL
      * @prop sourceConnectionId — Filters catalog entries by source connection
      * @prop selectedIds        — (bindable) Array of selected CatalogEntry IDs
      * @prop entries            — (bindable) Loaded catalog entries (exposed for parent access)
-     * @prop columns            — Column keys for the browse table (default: name, dataType, labels)
-     * @prop selectedColumnsDisplay — Column keys for the selected tags panel (default: name, dataType, labels)
+     * @prop columns            — Column keys for the browse table
+     * @prop selectedColumnsDisplay — Column keys for the selected tags panel
      * @prop emptyMessage       — Message when no entries match the source connection
      * @prop onSelectionChange  — Called when selection changes (receives full selectedIds array)
      * @prop onRemove           — Called when a single entry is removed from the selection
      */
-    import { untrack } from 'svelte';
     import { DataCatalogClient } from '@industream/datacatalog-client';
     import type { CatalogEntry } from '@industream/datacatalog-client/dto';
     import { DEFAULT_COLUMNS, DEFAULT_SELECTED_COLUMNS_DISPLAY } from './picker-column-helpers';
+    import {
+        PAGE_SIZE,
+        MATCH_ALL,
+        mergePage,
+        deduplicateEntries,
+        normalizeQuery,
+        isAllLoaded,
+        hasMore,
+        remainingCount,
+        upsertSelectedCache,
+        resolveSelected
+    } from './picker-pagination';
     import TagBrowser from './TagBrowser.svelte';
     import SelectedTags from './SelectedTags.svelte';
 
-    const MAX_ENTRIES = 100;
-
     interface Props {
         dcApiUrl?: string;
         sourceConnectionId?: string;
@@ -58,33 +67,46 @@
     }: Props = $props();
 
     let loading = $state(false);
+    let loadingMore = $state(false);
     let searching = $state(false);
     let error = $state('');
-    let tooMany = $state(false);
+    let loadMoreError = $state('');
+    let totalCount = $state(0);
+    let currentQuery = $state(MATCH_ALL);
+    let offset = $state(0);
 
-    /** Abort controller for cancelling in-flight search requests */
-    let searchAbort: AbortController | null = null;
-
-    /** Cached client instance — recreated when dcApiUrl changes */
-    let client: DataCatalogClient | null = null;
-    let clientUrl = '';
+    /**
+     * Metadata for selected tags, kept across reloads/searches so SelectedTags always
+     * renders. Invariant: every mutation of this Map is paired with an `entries`
+     * reassignment, so the `selectedEntries` derived (which reads both) re-runs and picks
+     * up the change. Do NOT upsert the cache without also reassigning `entries`.
+     */
+    let selectedCache = new Map<string, CatalogEntry>();
 
     /**
-     * Deduplicate entries by id — keep only the first occurrence.
+     * Best-effort total for the current query. The server's search response MAY omit
+     * `totalCount`; when it does, we must not silently cap pagination at one page.
+     * Fall back to: a full page → "at least one more page" (loaded + PAGE_SIZE) so the
+     * load-more affordance stays available; a short page → exactly what is loaded (the end).
      */
-    function deduplicateEntries(items: CatalogEntry[]): CatalogEntry[] {
-        const seen = new Set<string>();
-        const result: CatalogEntry[] = [];
-        for (const item of items) {
-            const id = item.id;
-            if (!seen.has(id)) {
-                seen.add(id);
-                result.push(item);
-            }
-        }
-        return result;
+    function estimateTotal(reported: number | undefined, loadedCount: number, lastPageLen: number): number {
+        if (reported != null) return reported;
+        return lastPageLen < PAGE_SIZE ? loadedCount : loadedCount + PAGE_SIZE;
     }
 
+    /** Abort controller for cancelling in-flight requests. */
+    let fetchAbort: AbortController | null = null;
+
+    /** Cached client instance — recreated when dcApiUrl changes. */
+    let client: DataCatalogClient | null = null;
+    let clientUrl = '';
+
+    /** Whether search runs server-side (large catalog) or client-side (everything loaded). */
+    const serverSearch = $derived(!isAllLoaded(MATCH_ALL, entries.length, totalCount) || currentQuery !== MATCH_ALL);
+
+    /** Resolved entries for the selected-tags panel: window first, cache fallback. */
+    const selectedEntries = $derived(resolveSelected(entries, selectedCache, selectedIds));
+
     function getClient(apiUrl: string): DataCatalogClient {
         if (client && clientUrl === apiUrl) return client;
         client = new DataCatalogClient({ baseUrl: apiUrl });
@@ -96,135 +118,124 @@
         if (dcApiUrl) {
             loadInitial(dcApiUrl, sourceConnectionId);
         } else {
+            fetchAbort?.abort();
             entries = [];
+            totalCount = 0;
             error = '';
-            tooMany = false;
         }
     });
 
-    /**
-     * Initial load — fetch MAX_ENTRIES + 1 to detect overflow.
-     * If ≤ MAX_ENTRIES items, stay in client mode.
-     * If more, switch to server search mode.
-     */
-    async function loadInitial(apiUrl: string, connId?: string) {
-        // Cancel any in-flight search
-        searchAbort?.abort();
-        searchAbort = null;
+    /** (Re)load page 1 for the match-all query. Never leaves the table blank on success. */
+    async function loadInitial(apiUrl: string, connId: string = '') {
+        fetchAbort?.abort();
+        const abort = new AbortController();
+        fetchAbort = abort;
 
         loading = true;
         error = '';
-        tooMany = false;
+        loadMoreError = '';
+        currentQuery = MATCH_ALL;
+        offset = 0;
         entries = [];
+        totalCount = 0;
 
         try {
             const dc = getClient(apiUrl);
-            const result = await dc.catalogEntries.search('%', {
+            const result = await dc.catalogEntries.search(MATCH_ALL, {
                 sourceConnectionIds: connId ? [connId] : [],
-                limit: MAX_ENTRIES + 1
-            });
-            const items = result.items ?? [];
-            const uniqueItems = deduplicateEntries(items);
-
-            if (uniqueItems.length > MAX_ENTRIES) {
-                // Too many for client-side filtering → server-search mode. Still show the
-                // first MAX_ENTRIES as a browsable PREVIEW (with a "not all displayed"
-                // warning in TagBrowser) instead of a blank "type to search" screen.
-                tooMany = true;
-                entries = uniqueItems.slice(0, MAX_ENTRIES);
-            } else {
-                tooMany = false;
-                entries = uniqueItems;
-            }
+                limit: PAGE_SIZE,
+                offset: 0
+            }, abort.signal);
+            if (abort.signal.aborted) return;
+            const items = deduplicateEntries(result.items ?? []);
+            entries = items;
+            totalCount = estimateTotal(result.totalCount, items.length, items.length);
+            offset = items.length;
+            upsertSelectedCache(selectedCache, items);
         } catch (err: any) {
-            error = err.message ?? 'Failed to load catalog entries';
+            if (abort.signal.aborted) return;
+            error = err?.message ?? 'Failed to load catalog entries';
             entries = [];
         } finally {
-            loading = false;
+            if (!abort.signal.aborted) loading = false;
         }
     }
 
-    /**
-     * Server-side search — called by TagBrowser when in server mode (tooMany).
-     * Cancels previous in-flight request via AbortSignal.
-     */
-    function handleServerSearch(query: string) {
+    /** Server-side search (large catalogs). Resets pagination for the new query. */
+    function handleServerSearch(rawQuery: string) {
         if (!dcApiUrl) return;
-
-        // Cancel previous in-flight search
-        searchAbort?.abort();
-
-        if (!query) {
-            // Empty query in too-many mode → restore the first-N preview.
-            searching = false;
-            loadInitial(dcApiUrl, sourceConnectionId);
-            return;
-        }
-
+        fetchAbort?.abort();
         const abort = new AbortController();
-        searchAbort = abort;
+        fetchAbort = abort;
+
+        const query = normalizeQuery(rawQuery);
+        currentQuery = query;
+        offset = 0;
+        entries = [];
+        totalCount = 0;
+        error = '';
+        loadMoreError = '';
         searching = true;
 
         const dc = getClient(dcApiUrl);
         dc.catalogEntries.search(query, {
             sourceConnectionIds: sourceConnectionId ? [sourceConnectionId] : [],
-            limit: MAX_ENTRIES
+            limit: PAGE_SIZE,
+            offset: 0
         }, abort.signal)
             .then((result) => {
                 if (abort.signal.aborted) return;
-                const items = result.items ?? [];
-                entries = deduplicateEntries(items);
+                const items = deduplicateEntries(result.items ?? []);
+                entries = items;
+                totalCount = estimateTotal(result.totalCount, items.length, items.length);
+                offset = items.length;
+                upsertSelectedCache(selectedCache, items);
                 searching = false;
             })
-            .catch((err) => {
+            .catch((err: any) => {
                 if (abort.signal.aborted) return;
-                error = err.message ?? 'Search failed';
+                error = err?.message ?? 'Search failed';
                 entries = [];
                 searching = false;
             });
     }
 
-    /**
-     * Full details of the SELECTED entries, fetched by ID — decoupled from the browse
-     * list (`entries`), which is empty in `tooMany` mode and only holds the current
-     * page/search otherwise. Without this, the SelectedTags panel can only render
-     * selected tags that happen to be in `entries` → on large sources it shows
-     * "No tags selected" / loses the selection visually despite a correct count.
-     */
-    let selectedEntries = $state<CatalogEntry[]>([]);
+    /** Append the next page for the current query. No-op when everything is loaded. */
+    function handleLoadMore() {
+        if (!dcApiUrl || loadingMore || loading || searching) return;
+        if (!hasMore(entries.length, totalCount)) return;
 
-    $effect(() => {
-        const ids = [...selectedIds];   // reactive dependency
-        const url = dcApiUrl;           // reactive dependency
-
-        if (!url || ids.length === 0) {
-            selectedEntries = [];
-            return;
-        }
-
-        // Reuse already-resolved entries that are still selected; fetch only the missing.
-        const idSet = new Set(ids);
-        const current = untrack(() => selectedEntries);
-        const keep = current.filter((e) => idSet.has(e.id));
-        const have = new Set(keep.map((e) => e.id));
-        const missing = ids.filter((id) => !have.has(id));
+        fetchAbort?.abort();
+        const abort = new AbortController();
+        fetchAbort = abort;
 
-        if (missing.length === 0) {
-            if (keep.length !== current.length) selectedEntries = keep;
-            return;
-        }
+        loadingMore = true;
+        loadMoreError = '';
+        const startOffset = offset;
 
-        // get() auto-switches to POST /catalog-entries/query for large id lists (URI-safe).
-        getClient(url).catalogEntries
-            .get({ ids: missing })
-            .then((res) => {
-                const latest = new Set(untrack(() => selectedIds));
-                selectedEntries = [...keep, ...(res.items ?? [])].filter((e) => latest.has(e.id));
+        const dc = getClient(dcApiUrl);
+        dc.catalogEntries.search(currentQuery, {
+            sourceConnectionIds: sourceConnectionId ? [sourceConnectionId] : [],
+            limit: PAGE_SIZE,
+            offset: startOffset
+        }, abort.signal)
+            .then((result) => {
+                if (abort.signal.aborted) return;
+                const items = result.items ?? [];
+                entries = mergePage(entries, items);
+                totalCount = estimateTotal(result.totalCount, entries.length, items.length);
+                offset = entries.length;
+                upsertSelectedCache(selectedCache, items);
+                loadingMore = false;
             })
-            .catch(() => {
-                // Keep what we have; SelectedTags falls back to the browse list below.
+            .catch((err: any) => {
+                if (abort.signal.aborted) return;
+                loadMoreError = err?.message ?? 'Failed to load more entries';
+                loadingMore = false;
             });
-    });
+    }
+
+    const remaining = $derived(remainingCount(entries.length, totalCount));
 </script>
 
 <TagBrowser
@@ -232,17 +243,22 @@
     bind:selectedIds
     {columns}
     {loading}
+    {loadingMore}
     {error}
+    {loadMoreError}
     {emptyMessage}
-    {onSelectionChange}
-    onSearch={tooMany ? handleServerSearch : null}
+    {totalCount}
+    {remaining}
+    serverSearch={serverSearch}
     {searching}
-    {tooMany}
+    {onSelectionChange}
+    onSearch={handleServerSearch}
+    onLoadMore={handleLoadMore}
 />
 
 {#if selectedIds.length > 0}
     <SelectedTags
-        entries={selectedEntries.length > 0 ? selectedEntries : entries}
+        entries={selectedEntries}
         bind:selectedIds
         columns={selectedColumnsDisplay}
         {onRemove}