@internetarchive/collection-browser 1.14.17-alpha.2 → 1.14.17-alpha.21

package/src/data-source/collection-browser-data-source.ts

@@ -12,13 +12,13 @@ import {
   SearchResult,
   SearchType,
 } from '@internetarchive/search-service';
-import type { MediaType } from '@internetarchive/field-parsers';
 import {
   prefixFilterAggregationKeys,
   type FacetBucket,
   type PrefixFilterType,
-  type TileModel,
+  TileModel,
   PrefixFilterCounts,
+  RequestKind,
 } from '../models';
 import type {
   CollectionBrowserSearchInterface,
@@ -27,8 +27,6 @@ import type {
 } from './models';
 import { sha1 } from '../utils/sha1';
 
-type RequestKind = 'full' | 'hits' | 'aggregations';
-
 export interface CollectionBrowserDataSourceInterface
   extends ReactiveController {
   /**
@@ -44,6 +42,12 @@ export interface CollectionBrowserDataSourceInterface
    */
   readonly canPerformSearch: boolean;
 
+  /**
+   * Whether the end of the set of results for the current query state has been
+   * encountered (i.e., the last page of results).
+   */
+  readonly endOfDataReached: boolean;
+
   /**
    * A string key compactly representing the current full search state, which can
    * be used to determine, e.g., when a new search is required or whether an arriving
@@ -128,6 +132,20 @@ export interface CollectionBrowserDataSourceInterface
    */
   readonly uncheckedTileModels: TileModel[];
 
+  /**
+   * A Promise which, after each query change, resolves once the fetches for the initial
+   * search have completed. Waits for *both* the hits and aggregations fetches to finish.
+   *
+   * Ensure you await this component's `updateComplete` promise before awaiting this
+   * one, to ensure you do not await an obsolete promise from the previous update.
+   */
+  readonly initialSearchComplete: Promise<boolean>;
+
+  /**
+   * Resets the data source to its empty state, with no result pages, aggregations, etc.
+   */
+  reset(): void;
+
   /**
    * Adds the given page of tile models to the data source.
    * If the given page number already exists, that page will be overwritten.
@@ -191,11 +209,6 @@ export interface CollectionBrowserDataSourceInterface
    */
   setPageSize(pageSize: number): void;
 
-  /**
-   * Resets the data source to its empty state, with no result pages, aggregations, etc.
-   */
-  reset(): void;
-
   /**
    * Notifies the data source that a query change has occurred, which may trigger a data
    * reset & new fetches.
@@ -242,8 +255,22 @@ export class CollectionBrowserDataSource
    */
   private pageFetchesInProgress: Record<string, Set<number>> = {};
 
+  /**
+   * A record of the query key used for the last search.
+   * If this changes, we need to load new results.
+   */
+  private previousQueryKey: string = '';
+
+  private id = Math.random();
+
+  /**
+   * @inheritdoc
+   */
   totalResults = 0;
 
+  /**
+   * @inheritdoc
+   */
   endOfDataReached = false;
 
   /**
@@ -287,6 +314,26 @@ export class CollectionBrowserDataSource
   prefixFilterCountMap: Partial<Record<PrefixFilterType, PrefixFilterCounts>> =
     {};
 
+  /**
+   * Internal property to store the `resolve` function for the most recent
+   * `initialSearchComplete` promise, allowing us to resolve it at the appropriate time.
+   */
+  private _initialSearchCompleteResolver!: (val: boolean) => void;
+
+  /**
+   * Internal property to store the private value backing the `initialSearchComplete` getter.
+   */
+  private _initialSearchCompletePromise: Promise<boolean> = new Promise(res => {
+    this._initialSearchCompleteResolver = res;
+  });
+
+  /**
+   * @inheritdoc
+   */
+  get initialSearchComplete(): Promise<boolean> {
+    return this._initialSearchCompletePromise;
+  }
+
   constructor(
     /** The host element to which this controller should attach listeners */
     private readonly host: ReactiveControllerHost &
@@ -297,6 +344,30 @@ export class CollectionBrowserDataSource
     this.host.addController(this as CollectionBrowserDataSourceInterface);
   }
 
+  hostUpdate(): void {
+    // This reactive controller hook is run whenever the host component (collection-browser) performs an update.
+    // We check whether the host's state has changed in a way which should trigger a reset & new results fetch.
+
+    // Only the currently-installed data source should react to the update
+    if (this.host.dataSource !== this) return;
+
+    // Can't perform searches without a search service
+    if (!this.host.searchService) return;
+
+    // We should only reset if part of the full query state has changed
+    const queryKeyChanged = this.pageFetchQueryKey !== this.previousQueryKey;
+    console.log('query keys', this.pageFetchQueryKey, this.previousQueryKey);
+    if (!queryKeyChanged) return;
+
+    // We should only reset if either:
+    //  (a) our state permits a valid search, or
+    //  (b) we have a blank query that we want to show empty results for
+    const shouldShowEmptyQueryResults = this.host.clearResultsOnEmptyQuery && this.host.baseQuery === '';
+    if (!(this.canPerformSearch || shouldShowEmptyQueryResults)) return;
+
+    this.handleQueryChange();
+  }
+
   /**
    * @inheritdoc
    */
@@ -304,6 +375,26 @@ export class CollectionBrowserDataSource
     return this.numTileModels;
   }
 
+  /**
+   * @inheritdoc
+   */
+  reset(): void {
+    this.pages = {};
+    this.aggregations = {};
+    this.yearHistogramAggregation = undefined;
+    this.pageFetchesInProgress = {};
+    this.pageElements = undefined;
+    this.parentCollections = [];
+    this.prefixFilterCountMap = {};
+
+    this.offset = 0;
+    this.numTileModels = 0;
+    this.totalResults = 0;
+    this.endOfDataReached = false;
+
+    this.host.requestUpdate();
+  }
+
   /**
    * @inheritdoc
    */
@@ -343,6 +434,9 @@ export class CollectionBrowserDataSource
     return this.pages[pageNum]?.[indexOnPage];
   }
 
+  /**
+   * @inheritdoc
+   */
   indexOf(tile: TileModel): number {
     return Object.values(this.pages).flat().indexOf(tile);
   }
@@ -355,35 +449,25 @@ export class CollectionBrowserDataSource
     this.pageSize = pageSize;
   }
 
-  /**
-   * @inheritdoc
-   */
-  reset(): void {
-    this.pages = {};
-    this.aggregations = {};
-    this.yearHistogramAggregation = undefined;
-    this.pageFetchesInProgress = {};
-    this.pageElements = undefined;
-    this.parentCollections = [];
-    this.prefixFilterCountMap = {};
-
-    this.offset = 0;
-    this.numTileModels = 0;
-    this.totalResults = 0;
-    this.endOfDataReached = false;
-
-    this.host.requestUpdate();
-  }
-
   /**
    * @inheritdoc
    */
   async handleQueryChange(): Promise<void> {
     this.reset();
+
+    // Reset the `initialSearchComplete` promise with a new value for the imminent search
+    this._initialSearchCompletePromise = new Promise(res => {
+      this._initialSearchCompleteResolver = res;
+    });
+
+    // Fire the initial page & facet requests
     await Promise.all([
       this.doInitialPageFetch(),
       this.host.suppressFacets ? null : this.fetchFacets(),
     ]);
+
+    // Resolve the `initialSearchComplete` promise for this search
+    this._initialSearchCompleteResolver(true);
   }
 
   /**
@@ -401,26 +485,35 @@ export class CollectionBrowserDataSource
       ])
     );
     this.host.requestUpdate();
+    this.host.refreshVisibleResults();
   }
 
   /**
    * @inheritdoc
    */
-  checkAllTiles(): void {
-    this.map(model => ({ ...model, checked: true }));
-  }
+  checkAllTiles = (): void => {
+    this.map(model => {
+      const cloned = model.clone();
+      cloned.checked = true;
+      return cloned;
+    });
+  };
 
   /**
    * @inheritdoc
    */
-  uncheckAllTiles(): void {
-    this.map(model => ({ ...model, checked: false }));
-  }
+  uncheckAllTiles = (): void => {
+    this.map(model => {
+      const cloned = model.clone();
+      cloned.checked = false;
+      return cloned;
+    });
+  };
 
   /**
    * @inheritdoc
    */
-  removeCheckedTiles(): void {
+  removeCheckedTiles = (): void => {
     // To make sure our data source remains page-aligned, we will offset our data source by
     // the number of removed tiles, so that we can just add the offset when the infinite
     // scroller queries for cell contents.
@@ -458,7 +551,8 @@ export class CollectionBrowserDataSource
     this.pages = newPages;
     this.numTileModels -= numChecked;
     this.host.requestUpdate();
-  }
+    this.host.refreshVisibleResults();
+  };
 
   /**
    * @inheritdoc
@@ -526,15 +620,15 @@ export class CollectionBrowserDataSource
    *  - Any currently-applied prefix filters
    *  - The current sort options
    *
-   * This lets us keep track of queries so we don't persist data that's
-   * no longer relevant.
+   * This lets us internally keep track of queries so we don't persist data that's
+   * no longer relevant. Not meant to be human-readable.
    */
   get pageFetchQueryKey(): string {
-    const profileKey = `${this.host.withinProfile}--${this.host.profileElement}`;
+    const profileKey = `pf;${this.host.withinProfile}--pe;${this.host.profileElement}`;
     const pageTarget = this.host.withinCollection ?? profileKey;
     const sortField = this.host.sortParam?.field ?? 'none';
     const sortDirection = this.host.sortParam?.direction ?? 'none';
-    return `${this.fullQuery}-${pageTarget}-${this.host.searchType}-${sortField}-${sortDirection}`;
+    return `fq:${this.fullQuery}-pt:${pageTarget}-st:${this.host.searchType}-sf:${sortField}-sd:${sortDirection}`;
   }
 
   /**
@@ -810,8 +904,10 @@ export class CollectionBrowserDataSource
    * the current search state.
    */
   private async fetchFacets(): Promise<void> {
+    console.log('fetchFacets', this.id, this.host.profileElement);
     const trimmedQuery = this.host.baseQuery?.trim();
     if (!this.canPerformSearch) return;
+    console.log('will actually fetch facets');
 
     const { facetFetchQueryKey } = this;
 
@@ -843,7 +939,10 @@ export class CollectionBrowserDataSource
     // likely no longer valid for the newer query.
     const queryChangedSinceFetch =
       facetFetchQueryKey !== this.facetFetchQueryKey;
-    if (queryChangedSinceFetch) return;
+    if (queryChangedSinceFetch) {
+      console.log('facet query has changed since fetch, returning. new/old:', this.facetFetchQueryKey, facetFetchQueryKey);
+      return;
+    }
 
     if (!success) {
       const errorMsg = searchResponse?.error?.message;
@@ -857,6 +956,7 @@ export class CollectionBrowserDataSource
         );
       }
 
+      this.host.setFacetsLoading(false);
       return;
     }
 
@@ -897,11 +997,13 @@ export class CollectionBrowserDataSource
   async fetchPage(pageNumber: number, numInitialPages = 1): Promise<void> {
     console.log(
       `fetchPage(${pageNumber})`,
+      this.id,
       this.canPerformSearch,
       this.hasPage(pageNumber),
       this.endOfDataReached,
       this.host.baseQuery,
-      JSON.stringify(this.host.selectedFacets)
+      JSON.stringify(this.host.selectedFacets),
+      this.pageFetchQueryKey
     );
     const trimmedQuery = this.host.baseQuery?.trim();
     if (!this.canPerformSearch) return;
@@ -930,6 +1032,7 @@ export class CollectionBrowserDataSource
       pageFetches.add(pageNumber + i);
     }
     this.pageFetchesInProgress[pageFetchQueryKey] = pageFetches;
+    this.previousQueryKey = pageFetchQueryKey;
 
     const sortParams = this.host.sortParam ? [this.host.sortParam] : [];
     const params: SearchParams = {
@@ -990,6 +1093,10 @@ export class CollectionBrowserDataSource
     }
 
     if (this.host.withinCollection) {
+      console.log(
+        'host is within collection, setting collection info:',
+        success.response.collectionExtraInfo
+      );
       this.collectionExtraInfo = success.response.collectionExtraInfo;
 
       // For collections, we want the UI to respect the default sort option
@@ -1004,10 +1111,17 @@ export class CollectionBrowserDataSource
     } else if (this.host.withinProfile) {
       console.log(
         'host is within profile, setting acct info:',
-        success.response.accountExtraInfo
+        success.response.accountExtraInfo,
+        success.response.pageElements
       );
       this.accountExtraInfo = success.response.accountExtraInfo;
       this.pageElements = success.response.pageElements;
+    } else {
+      console.log(
+        'not within profile/collxn',
+        this.host.withinCollection,
+        this.host.withinProfile
+      );
     }
 
     const { results, collectionTitles } = success.response;
@@ -1035,6 +1149,7 @@ export class CollectionBrowserDataSource
     // temporary estimates based on pages rendered so far).
     const resultCountDiscrepancy = numRows - results.length;
     if (resultCountDiscrepancy > 0) {
+      console.log('End of data reached');
       this.endOfDataReached = true;
       this.host.setTileCount(this.totalResults);
     }
@@ -1063,56 +1178,7 @@ export class CollectionBrowserDataSource
     const tiles: TileModel[] = [];
     results?.forEach(result => {
       if (!result.identifier) return;
-
-      let loginRequired = false;
-      let contentWarning = false;
-      // Check if item and item in "modifying" collection, setting above flags
-      if (
-        result.collection?.values.length &&
-        result.mediatype?.value !== 'collection'
-      ) {
-        for (const collection of result.collection?.values ?? []) {
-          if (collection === 'loggedin') {
-            loginRequired = true;
-            if (contentWarning) break;
-          }
-          if (collection === 'no-preview') {
-            contentWarning = true;
-            if (loginRequired) break;
-          }
-        }
-      }
-
-      tiles.push({
-        averageRating: result.avg_rating?.value,
-        checked: false,
-        collections: result.collection?.values ?? [],
-        collectionFilesCount: result.collection_files_count?.value ?? 0,
-        collectionSize: result.collection_size?.value ?? 0,
-        commentCount: result.num_reviews?.value ?? 0,
-        creator: result.creator?.value,
-        creators: result.creator?.values ?? [],
-        dateAdded: result.addeddate?.value,
-        dateArchived: result.publicdate?.value,
-        datePublished: result.date?.value,
-        dateReviewed: result.reviewdate?.value,
-        description: result.description?.values.join('\n'),
-        favCount: result.num_favorites?.value ?? 0,
-        href: this.collapseRepeatedQuotes(result.__href__?.value),
-        identifier: result.identifier,
-        issue: result.issue?.value,
-        itemCount: result.item_count?.value ?? 0,
-        mediatype: this.getMediatype(result),
-        snippets: result.highlight?.values ?? [],
-        source: result.source?.value,
-        subjects: result.subject?.values ?? [],
-        title: result.title?.value ?? '',
-        volume: result.volume?.value,
-        viewCount: result.downloads?.value ?? 0,
-        weeklyViewCount: result.week?.value,
-        loginRequired,
-        contentWarning,
-      });
+      tiles.push(new TileModel(result));
     });
     this.addPage(pageNumber, tiles);
     const visiblePages = this.host.currentVisiblePageNumbers;
@@ -1122,44 +1188,6 @@ export class CollectionBrowserDataSource
     }
   }
 
-  /**
-   * Returns the mediatype string for the given search result, taking into account
-   * the special `favorited_search` hit type.
-   * @param result The search result to extract a mediatype from
-   */
-  private getMediatype(result: SearchResult): MediaType {
-    /**
-     * hit_type == 'favorited_search' is basically a new hit_type
-     * - we are getting from PPS.
-     * - which gives response for fav- collection
-     * - having favorited items like account/collection/item etc..
-     * - as user can also favorite a search result (a search page)
-     * - so we need to have response (having fav- items and fav- search results)
-     *
-     * if backend hit_type == 'favorited_search'
-     * - let's assume a "search" as new mediatype
-     */
-    if (result?.rawMetadata?.hit_type === 'favorited_search') {
-      return 'search';
-    }
-
-    return result.mediatype?.value ?? 'data';
-  }
-
-  /**
-   * Returns the input string, but removing one set of quotes from all instances of
-   * ""clauses wrapped in two sets of quotes"". This assumes the quotes are already
-   * URL-encoded.
-   *
-   * This should be a temporary measure to address the fact that the __href__ field
-   * sometimes acquires extra quotation marks during query rewriting. Once there is a
-   * full Lucene parser in place that handles quoted queries correctly, this can likely
-   * be removed.
-   */
-  private collapseRepeatedQuotes(str?: string): string | undefined {
-    return str?.replace(/%22%22(?!%22%22)(.+?)%22%22/g, '%22$1%22');
-  }
-
   /**
    * Fetches the aggregation buckets for the given prefix filter type.
    */

package/src/data-source/models.ts

@@ -1,5 +1,6 @@
 import type {
   CollectionExtraInfo,
+  PageElementName,
   PageType,
   SearchServiceInterface,
   SearchType,
@@ -7,6 +8,7 @@ import type {
   SortParam,
 } from '@internetarchive/search-service';
 import type { SelectedFacets, SortField } from '../models';
+import type { CollectionBrowserDataSourceInterface } from './collection-browser-data-source';
 
 /**
  * A Map from collection identifiers to their corresponding collection titles.
@@ -30,7 +32,7 @@ export type PageSpecifierParams = {
    * Which specific elements of a profile page to fetch. Corresponds to individual tab data
    * (e.g., "uploads", "reviews", ...)
    */
-  pageElements?: string[];
+  pageElements?: PageElementName[];
 };
 
 /**
@@ -40,7 +42,7 @@ export interface CollectionBrowserQueryState {
   baseQuery?: string;
   withinCollection?: string;
   withinProfile?: string;
-  profileElement?: string;
+  profileElement?: PageElementName;
   searchType: SearchType;
   selectedFacets?: SelectedFacets;
   minSelectedDate?: string;
@@ -63,6 +65,8 @@ export interface CollectionBrowserSearchInterface
   readonly suppressFacets?: boolean;
   readonly initialPageNumber: number;
   readonly currentVisiblePageNumbers: number[];
+  readonly clearResultsOnEmptyQuery?: boolean;
+  readonly dataSource: CollectionBrowserDataSourceInterface;
 
   getSessionId(): Promise<string>;
   setSearchResultsLoading(loading: boolean): void;

package/src/manage/manage-bar.ts

@@ -6,7 +6,7 @@ import { when } from 'lit/directives/when.js';
 export interface ManageableItem {
   identifier: string;
   title?: string;
-  date?: string;
+  dateStr?: string;
 }
 
 @customElement('manage-bar')