@internetarchive/collection-browser 0.4.0 → 0.4.2

package/src/collection-browser.ts

@@ -17,6 +17,10 @@ import type {
 } from '@internetarchive/infinite-scroller';
 import {
   Aggregation,
+  Bucket,
+  FilterConstraint,
+  FilterMap,
+  FilterMapBuilder,
   SearchParams,
   SearchResult,
   SearchServiceInterface,
@@ -47,6 +51,9 @@ import {
   CollectionDisplayMode,
   FacetOption,
   FacetBucket,
+  PrefixFilterType,
+  PrefixFilterCounts,
+  prefixFilterAggregationKeys,
 } from './models';
 import {
   RestorationStateHandlerInterface,
@@ -92,8 +99,6 @@ export class CollectionBrowser
 
   @property({ type: String }) sortDirection: SortDirection | null = null;
 
-  @property({ type: String }) dateRangeQueryClause?: string;
-
   @property({ type: Number }) pageSize = 50;
 
   @property({ type: Object }) resizeObserver?: SharedResizeObserverInterface;
@@ -177,6 +182,10 @@ export class CollectionBrowser
 
   @state() private placeholderType: PlaceholderType = null;
 
+  @state() private prefixFilterCountMap: Partial<
+    Record<PrefixFilterType, PrefixFilterCounts>
+  > = {};
+
   @query('#content-container') private contentContainer!: HTMLDivElement;
 
   private languageCodeHandler = new LanguageCodeHandler();
@@ -323,6 +332,10 @@ export class CollectionBrowser
   }
 
   private get collectionBrowserTemplate() {
+    const shouldShowSearching =
+      this.searchResultsLoading || this.totalResults === undefined;
+    const resultsCount = this.totalResults?.toLocaleString();
+    const resultsLabel = this.totalResults === 1 ? 'Result' : 'Results';
     return html`<div
         id="left-column"
         class="column${this.isResizeToMobile ? ' preload' : ''}"
@@ -331,12 +344,10 @@ export class CollectionBrowser
           ${this.mobileView ? this.mobileFacetsTemplate : nothing}
           <div id="results-total">
             <span id="big-results-count">
-              ${this.totalResults !== undefined
-                ? this.totalResults.toLocaleString()
-                : '-'}
+              ${shouldShowSearching ? html`Searching&hellip;` : resultsCount}
             </span>
             <span id="big-results-label">
-              ${this.totalResults === 1 ? 'Result' : 'Results'}
+              ${shouldShowSearching ? nothing : resultsLabel}
             </span>
           </div>
         </div>
@@ -376,6 +387,7 @@ export class CollectionBrowser
         .displayMode=${this.displayMode}
         .selectedTitleFilter=${this.selectedTitleFilter}
         .selectedCreatorFilter=${this.selectedCreatorFilter}
+        .prefixFilterCountMap=${this.prefixFilterCountMap}
         .resizeObserver=${this.resizeObserver}
         @sortChanged=${this.userChangedSort}
         @displayModeChanged=${this.displayModeChanged}
@@ -423,6 +435,9 @@ export class CollectionBrowser
 
     if (!sortField) return;
     this.sortParam = { field: sortField, direction: this.sortDirection };
+
+    // Lazy-load the alphabet counts for title/creator sort bar as needed
+    this.updatePrefixFiltersForCurrentSort();
   }
 
   private displayModeChanged(
@@ -541,7 +556,8 @@ export class CollectionBrowser
         .collectionNameCache=${this.collectionNameCache}
         .languageCodeHandler=${this.languageCodeHandler}
         .showHistogramDatePicker=${this.showHistogramDatePicker}
-        .fullQuery=${this.fullQuery}
+        .query=${this.filteredQuery}
+        .filterMap=${this.filterMap}
         .modalManager=${this.modalManager}
         ?collapsableFacets=${this.mobileView}
         ?facetsLoading=${this.facetDataLoading}
@@ -576,21 +592,6 @@ export class CollectionBrowser
     `;
   }
 
-  private get queryDebuggingTemplate() {
-    return html`
-      <div>
-        <ul>
-          <li>Base Query: ${this.baseQuery}</li>
-          <li>Facet Query: ${this.facetQuery}</li>
-          <li>Sort Filter Query: ${this.sortFilterQueries}</li>
-          <li>Date Range Query: ${this.dateRangeQueryClause}</li>
-          <li>Sort: ${this.sortParam?.field} ${this.sortParam?.direction}</li>
-          <li>Full Query: ${this.fullQuery}</li>
-        </ul>
-      </div>
-    `;
-  }
-
   private histogramDateRangeUpdated(
     e: CustomEvent<{
       minDate: string;
@@ -598,17 +599,21 @@ export class CollectionBrowser
     }>
   ) {
     const { minDate, maxDate } = e.detail;
-
     [this.minSelectedDate, this.maxSelectedDate] = [minDate, maxDate];
-    this.dateRangeQueryClause = `year:[${minDate} TO ${maxDate}]`;
 
-    if (this.dateRangeQueryClause) {
-      this.analyticsHandler?.sendEvent({
-        category: this.searchContext,
-        action: analyticsActions.histogramChanged,
-        label: this.dateRangeQueryClause,
-      });
+    this.analyticsHandler?.sendEvent({
+      category: this.searchContext,
+      action: analyticsActions.histogramChanged,
+      label: this.dateRangeQueryClause,
+    });
+  }
+
+  private get dateRangeQueryClause() {
+    if (!this.minSelectedDate || !this.maxSelectedDate) {
+      return undefined;
     }
+
+    return `year:[${this.minSelectedDate} TO ${this.maxSelectedDate}]`;
   }
 
   firstUpdated(): void {
@@ -638,13 +643,22 @@ export class CollectionBrowser
       changed.has('baseQuery') ||
       changed.has('titleQuery') ||
       changed.has('creatorQuery') ||
-      changed.has('dateRangeQueryClause') ||
+      changed.has('minSelectedDate') ||
+      changed.has('maxSelectedDate') ||
       changed.has('sortParam') ||
       changed.has('selectedFacets') ||
       changed.has('searchService')
     ) {
       this.handleQueryChange();
     }
+    if (
+      changed.has('baseQuery') ||
+      changed.has('minSelectedDate') ||
+      changed.has('maxSelectedDate') ||
+      changed.has('selectedFacets')
+    ) {
+      this.refreshLetterCounts();
+    }
     if (changed.has('selectedSort') || changed.has('sortDirection')) {
       const prevSortDirection = changed.get('sortDirection') as SortDirection;
       this.sendSortByAnalytics(prevSortDirection);
@@ -775,6 +789,9 @@ export class CollectionBrowser
     this.previousQueryKey = this.pageFetchQueryKey;
 
     this.dataSource = {};
+    this.totalResults = undefined;
+    this.aggregations = undefined;
+    this.fullYearsHistogramAggregation = undefined;
     this.pageFetchesInProgress = {};
     this.endOfDataReached = false;
     this.pagesToRender = this.initialPageNumber;
@@ -830,7 +847,6 @@ export class CollectionBrowser
     this.baseQuery = restorationState.baseQuery;
     this.titleQuery = restorationState.titleQuery;
     this.creatorQuery = restorationState.creatorQuery;
-    this.dateRangeQueryClause = restorationState.dateRangeQueryClause;
     this.sortParam = restorationState.sortParam ?? null;
     this.currentPage = restorationState.currentPage ?? 1;
     this.minSelectedDate = restorationState.minSelectedDate;
@@ -850,7 +866,6 @@ export class CollectionBrowser
       selectedFacets: this.selectedFacets ?? defaultSelectedFacets,
       baseQuery: this.baseQuery,
       currentPage: this.currentPage,
-      dateRangeQueryClause: this.dateRangeQueryClause,
       titleQuery: this.titleQuery,
       creatorQuery: this.creatorQuery,
       minSelectedDate: this.minSelectedDate,
@@ -867,6 +882,84 @@ export class CollectionBrowser
     this.searchResultsLoading = false;
   }
 
+  private get filterMap(): FilterMap {
+    const builder = new FilterMapBuilder();
+
+    // Add the date range, if applicable
+    if (this.minSelectedDate) {
+      builder.addFilter(
+        'year',
+        this.minSelectedDate,
+        FilterConstraint.GREATER_OR_EQUAL
+      );
+    }
+    if (this.maxSelectedDate) {
+      builder.addFilter(
+        'year',
+        this.maxSelectedDate,
+        FilterConstraint.LESS_OR_EQUAL
+      );
+    }
+
+    // Add any selected facets
+    if (this.selectedFacets) {
+      for (const [facetName, facetValues] of Object.entries(
+        this.selectedFacets
+      )) {
+        const { name, values } = this.prepareFacetForFetch(
+          facetName,
+          facetValues
+        );
+        for (const [value, bucket] of Object.entries(values)) {
+          let constraint;
+          if (bucket.state === 'selected') {
+            constraint = FilterConstraint.INCLUDE;
+          } else if (bucket.state === 'hidden') {
+            constraint = FilterConstraint.EXCLUDE;
+          }
+
+          if (constraint) {
+            builder.addFilter(name, value, constraint);
+          }
+        }
+      }
+    }
+
+    const filterMap = builder.build();
+
+    // TEMP: At present, the backend search engine incorrectly returns 0 results if
+    // the _first_ language filter contains a space, so let's try to avoid that if possible.
+    if (filterMap.language) {
+      for (const [value, constraint] of Object.entries(filterMap.language)) {
+        if (value.includes(' ')) {
+          // Delete and re-add this filter to make it the last one on the parent object
+          // (Technically this isn't in the standard, but most browser impls output
+          // object keys in the order they were added.)
+          delete filterMap.language[value];
+          filterMap.language[value] = constraint;
+        } else {
+          // As soon as we reach one without a space, we're done
+          break;
+        }
+      }
+    }
+
+    return filterMap;
+  }
+
+  /** The base query joined with any title/creator letter filters */
+  private get filteredQuery(): string | undefined {
+    if (!this.baseQuery) return undefined;
+    let filteredQuery = this.baseQuery;
+
+    const { sortFilterQueries } = this;
+    if (sortFilterQueries) {
+      filteredQuery += ` AND ${sortFilterQueries}`;
+    }
+
+    return filteredQuery;
+  }
+
   /** The full query, including year facets and date range clauses */
   private get fullQuery(): string | undefined {
     if (!this.baseQuery) return undefined;
@@ -886,6 +979,22 @@ export class CollectionBrowser
     return fullQuery;
   }
 
+  /** The full query without any title/creator letter filters */
+  private get fullQueryWithoutAlphaFilters(): string | undefined {
+    if (!this.baseQuery) return undefined;
+    let fullQuery = this.baseQuery;
+
+    const { facetQuery, dateRangeQueryClause } = this;
+
+    if (facetQuery) {
+      fullQuery += ` AND ${facetQuery}`;
+    }
+    if (dateRangeQueryClause) {
+      fullQuery += ` AND ${dateRangeQueryClause}`;
+    }
+    return fullQuery;
+  }
+
   /** The full query without any year facets or date range clauses */
   private get fullQueryWithoutDates(): string | undefined {
     if (!this.baseQuery) return undefined;
@@ -951,29 +1060,57 @@ export class CollectionBrowser
     facetName: string,
     facetValues: Record<string, FacetBucket>
   ): string {
-    const facetEntries = Object.entries(facetValues);
+    const { name: facetQueryName, values } = this.prepareFacetForFetch(
+      facetName,
+      facetValues
+    );
+    const facetEntries = Object.entries(values);
     if (facetEntries.length === 0) return '';
 
-    const facetQueryName =
-      facetName === 'lending' ? 'lending___status' : facetName;
     const facetValuesArray: string[] = [];
-
     for (const [key, facetData] of facetEntries) {
       const plusMinusPrefix = facetData.state === 'hidden' ? '-' : '';
+      facetValuesArray.push(`${plusMinusPrefix}"${key}"`);
+    }
+
+    const valueQuery = facetValuesArray.join(` OR `);
+    return `${facetQueryName}:(${valueQuery})`;
+  }
 
-      if (facetName === 'language') {
+  /**
+   * Handles some special pre-request normalization steps for certain facet types
+   * that require them.
+   *
+   * @param facetName The name of the facet type (e.g., 'language')
+   * @param facetValues An array of values for that facet type
+   */
+  private prepareFacetForFetch(
+    facetName: string,
+    facetValues: Record<string, FacetBucket>
+  ): { name: string; values: Record<string, FacetBucket> } {
+    let [normalizedName, normalizedValues] = [facetName, facetValues];
+
+    // The full "search engine" name of the lending field is "lending___status"
+    if (facetName === 'lending') {
+      normalizedName = 'lending___status';
+    }
+
+    // Language codes like "en-US|en-GB|en" need to be broken apart into individual values
+    if (facetName === 'language') {
+      normalizedValues = {};
+      for (const [facetValue, facetData] of Object.entries(facetValues)) {
         const languages =
-          this.languageCodeHandler.getCodeArrayFromCodeString(key);
-        for (const language of languages) {
-          facetValuesArray.push(`${plusMinusPrefix}"${language}"`);
+          this.languageCodeHandler.getCodeArrayFromCodeString(facetValue);
+        for (const lang of languages) {
+          normalizedValues[lang] = { ...facetData };
         }
-      } else {
-        facetValuesArray.push(`${plusMinusPrefix}"${key}"`);
       }
     }
 
-    const valueQuery = facetValuesArray.join(` OR `);
-    return `${facetQueryName}:(${valueQuery})`;
+    return {
+      name: normalizedName,
+      values: normalizedValues,
+    };
   }
 
   /**
@@ -1018,11 +1155,12 @@ export class CollectionBrowser
   }
 
   private async fetchFacets() {
-    if (!this.fullQuery) return;
+    if (!this.filteredQuery) return;
 
     const params: SearchParams = {
-      query: this.fullQuery,
+      query: this.filteredQuery,
       rows: 0,
+      filters: this.filterMap,
       // Fetch a few extra buckets beyond the 6 we show, in case some get suppressed
       aggregationsSize: 10,
       // Note: we don't need an aggregations param to fetch the default aggregations from the PPS.
@@ -1132,7 +1270,14 @@ export class CollectionBrowser
   }
 
   /**
-   * The query key is a string that uniquely identifies the current query
+   * The query key is a string that uniquely identifies the current search.
+   * It consists of:
+   *  - The current base query
+   *  - The current search type
+   *  - Any currently-applied facets
+   *  - Any currently-applied date range
+   *  - Any currently-applied prefix filters
+   *  - The current sort options
    *
    * This lets us keep track of queries so we don't persist data that's
    * no longer relevant.
@@ -1145,7 +1290,7 @@ export class CollectionBrowser
   private pageFetchesInProgress: Record<string, Set<number>> = {};
 
   async fetchPage(pageNumber: number) {
-    if (!this.fullQuery) return;
+    if (!this.filteredQuery) return;
 
     // if we already have data, don't fetch again
     if (this.dataSource[pageNumber]) return;
@@ -1162,10 +1307,11 @@ export class CollectionBrowser
 
     const sortParams = this.sortParam ? [this.sortParam] : [];
     const params: SearchParams = {
-      query: this.fullQuery,
+      query: this.filteredQuery,
       page: pageNumber,
       rows: this.pageSize,
       sort: sortParams,
+      filters: this.filterMap,
       aggregations: { omit: true },
     };
     const searchResponse = await this.searchService?.search(
@@ -1203,7 +1349,7 @@ export class CollectionBrowser
       }
     }
     const queryChangedSinceFetch =
-      searchQuery !== this.fullQuery || sortChanged;
+      searchQuery !== this.filteredQuery || sortChanged;
     if (queryChangedSinceFetch) return;
 
     const { results } = success.response;
@@ -1211,12 +1357,13 @@ export class CollectionBrowser
       this.preloadCollectionNames(results);
       this.updateDataSource(pageNumber, results);
     }
+
+    // When we reach the end of the data, we can set the infinite scroller's
+    // item count to the real total number of results (rather than the
+    // temporary estimates based on pages rendered so far).
     if (results.length < this.pageSize) {
       this.endOfDataReached = true;
-      // this updates the infinite scroller to show the actual size
-      if (this.infiniteScroller) {
-        this.infiniteScroller.itemCount = this.actualTileCount;
-      }
+      this.infiniteScroller.itemCount = this.totalResults;
     }
     this.pageFetchesInProgress[pageFetchQueryKey]?.delete(pageNumber);
     this.searchResultsLoading = false;
@@ -1320,6 +1467,73 @@ export class CollectionBrowser
     }
   }
 
+  /** Fetches the aggregation buckets for the given prefix filter type. */
+  private async fetchPrefixFilterBuckets(
+    filterType: PrefixFilterType
+  ): Promise<Bucket[]> {
+    if (!this.fullQueryWithoutAlphaFilters) return [];
+
+    const filterAggregationKey = prefixFilterAggregationKeys[filterType];
+    const params: SearchParams = {
+      query: this.fullQueryWithoutAlphaFilters,
+      rows: 0,
+      // Only fetch the firstTitle or firstCreator aggregation
+      aggregations: { simpleParams: [filterAggregationKey] },
+      // Fetch all 26 letter buckets
+      aggregationsSize: 26,
+    };
+
+    const searchResponse = await this.searchService?.search(
+      params,
+      this.searchType
+    );
+
+    return (searchResponse?.success?.response?.aggregations?.[
+      filterAggregationKey
+    ]?.buckets ?? []) as Bucket[];
+  }
+
+  /** Fetches and caches the prefix filter counts for the given filter type. */
+  private async updatePrefixFilterCounts(
+    filterType: PrefixFilterType
+  ): Promise<void> {
+    const buckets = await this.fetchPrefixFilterBuckets(filterType);
+    // Unpack the aggregation buckets into a simple map like { 'A': 50, 'B': 25, ... }
+    this.prefixFilterCountMap = { ...this.prefixFilterCountMap }; // Clone the object to trigger an update
+    this.prefixFilterCountMap[filterType] = buckets.reduce(
+      (acc: Record<string, number>, bucket: Bucket) => {
+        acc[(bucket.key as string).toUpperCase()] = bucket.doc_count;
+        return acc;
+      },
+      {}
+    );
+  }
+
+  /**
+   * Fetches and caches the prefix filter counts for the current sort type,
+   * provided it is one that permits prefix filtering. (If not, this does nothing).
+   */
+  private async updatePrefixFiltersForCurrentSort(): Promise<void> {
+    if (['title', 'creator'].includes(this.selectedSort)) {
+      const filterType = this.selectedSort as PrefixFilterType;
+      if (!this.prefixFilterCountMap[filterType]) {
+        this.updatePrefixFilterCounts(filterType);
+      }
+    }
+  }
+
+  /**
+   * Clears the cached letter counts for both title and creator, and
+   * fetches a new set of counts for whichever of them is the currently
+   * selected sort option (which may be neither).
+   *
+   * Call this whenever the counts are invalidated (e.g., by a query change).
+   */
+  private refreshLetterCounts(): void {
+    this.prefixFilterCountMap = {};
+    this.updatePrefixFiltersForCurrentSort();
+  }
+
   /*
    * Convert etree titles
    * "[Creator] Live at [Place] on [Date]" => "[Date]: [Place]"
@@ -1385,8 +1599,10 @@ export class CollectionBrowser
    * increase the number of pages to render and start fetching data for the new page
    */
   private scrollThresholdReached() {
-    this.pagesToRender += 1;
-    this.fetchPage(this.pagesToRender);
+    if (!this.endOfDataReached) {
+      this.pagesToRender += 1;
+      this.fetchPage(this.pagesToRender);
+    }
   }
 
   static styles = css`

package/src/collection-facets/more-facets-content.ts

@@ -17,6 +17,7 @@ import {
   SearchParams,
   SearchType,
   AggregationSortType,
+  FilterMap,
 } from '@internetarchive/search-service';
 import type { CollectionNameCacheInterface } from '@internetarchive/collection-name-cache';
 import type { ModalManagerInterface } from '@internetarchive/modal-manager';
@@ -44,7 +45,9 @@ export class MoreFacetsContent extends LitElement {
 
   @property({ type: String }) facetAggregationKey?: FacetOption;
 
-  @property({ type: String }) fullQuery?: string;
+  @property({ type: String }) query?: string;
+
+  @property({ type: Object }) filterMap?: FilterMap;
 
   @property({ type: Object }) modalManager?: ModalManagerInterface;
 
@@ -127,7 +130,8 @@ export class MoreFacetsContent extends LitElement {
     const aggregationsSize = 65535; // todo - do we want to have all the records at once?
 
     const params: SearchParams = {
-      query: this.fullQuery as string,
+      query: this.query as string,
+      filters: this.filterMap,
       aggregations,
       aggregationsSize,
       rows: 0, // todo - do we want server-side pagination with offset/page/limit flag?

package/src/collection-facets.ts

@@ -12,6 +12,7 @@ import { map } from 'lit/directives/map.js';
 import type {
   Aggregation,
   Bucket,
+  FilterMap,
   SearchServiceInterface,
   SearchType,
 } from '@internetarchive/search-service';
@@ -72,7 +73,9 @@ export class CollectionFacets extends LitElement {
 
   @property({ type: Boolean }) showHistogramDatePicker = false;
 
-  @property({ type: String }) fullQuery?: string;
+  @property({ type: String }) query?: string;
+
+  @property({ type: Object }) filterMap?: FilterMap;
 
   @property({ type: Object, attribute: false })
   modalManager?: ModalManagerInterface;
@@ -491,7 +494,8 @@ export class CollectionFacets extends LitElement {
         .analyticsHandler=${this.analyticsHandler}
         .facetKey=${facetGroup.key}
         .facetAggregationKey=${facetAggrKey}
-        .fullQuery=${this.fullQuery}
+        .query=${this.query}
+        .filterMap=${this.filterMap}
         .modalManager=${this.modalManager}
         .searchService=${this.searchService}
         .searchType=${this.searchType}

package/src/models.ts

@@ -119,6 +119,21 @@ export const MetadataFieldToSortField: {
   creatorSorter: SortField.creator,
 };
 
+/** A union of the fields that permit prefix filtering (e.g., alphabetical filtering) */
+export type PrefixFilterType = 'title' | 'creator';
+
+/** A map from prefixes (e.g., initial letters) to the number of items matching that prefix */
+export type PrefixFilterCounts = Record<string, number>;
+
+/**
+ * A map from prefix filter types to the corresponding aggregation keys
+ * that are needed to fetch the filter counts from the backend.
+ */
+export const prefixFilterAggregationKeys: Record<PrefixFilterType, string> = {
+  title: 'firstTitle',
+  creator: 'firstCreator',
+};
+
 export type FacetOption =
   | 'subject'
   | 'lending'

package/src/restoration-state-handler.ts

@@ -25,7 +25,6 @@ export interface RestorationState {
   selectedFacets: SelectedFacets;
   baseQuery?: string;
   currentPage?: number;
-  dateRangeQueryClause?: string;
   titleQuery?: string;
   creatorQuery?: string;
   minSelectedDate?: string;
@@ -142,8 +141,11 @@ export class RestorationStateHandler
       }
     }
 
-    if (state.dateRangeQueryClause) {
-      searchParams.append('and[]', state.dateRangeQueryClause);
+    if (state.minSelectedDate && state.maxSelectedDate) {
+      searchParams.append(
+        'and[]',
+        `year:[${state.minSelectedDate} TO ${state.maxSelectedDate}]`
+      );
     }
     if (state.titleQuery) {
       searchParams.append('and[]', state.titleQuery);
@@ -159,7 +161,8 @@ export class RestorationStateHandler
         page: state.currentPage,
         and: state.selectedFacets,
         not: state.selectedFacets,
-        dateRange: state.dateRangeQueryClause,
+        minDate: state.minSelectedDate,
+        maxDate: state.maxSelectedDate,
       },
       '',
       url
@@ -244,7 +247,6 @@ export class RestorationStateHandler
                 0,
                 maxDate.length - 1
               );
-              restorationState.dateRangeQueryClause = `year:${value}`;
             } else {
               this.setSelectedFacetState(
                 restorationState.selectedFacets,