@digigov/text-search 1.0.1 → 1.1.0

package/dist/src/hook.spec.ts

@@ -0,0 +1,289 @@
+import { act, renderHook } from '@testing-library/react-hooks';
+import useSearch from './hook';
+import documents from './test-utils/data.json';
+
+describe('loading', () => {
+  it('should start when searching', async () => {
+    const { result } = renderHook(() => useSearch(documents, 'Anastasia'));
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+  });
+  it('should stop after search is completed', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Anastasia')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+  });
+});
+
+describe('reset', () => {
+  it('should change the returned result back to the initial data', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Anastasia')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    act(() => result.current.reset());
+    expect(result.current.data).toEqual(documents);
+    expect(result.current.loading).toBe(false);
+  });
+});
+
+describe('search', () => {
+  it('should return the initial data when no query is passed', async () => {
+    const { result } = renderHook(() => useSearch(documents));
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual(documents);
+  });
+  it('should return an empty array when there are no relevant items', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'fajsgsdgsdgsd')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual([]);
+  });
+  it('should return an empty array when relevant field is not indexed', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Anastasia', {
+        indexing: {
+          fields: ['lastName'], // missing 'firstName'
+        },
+      })
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual([]);
+  });
+  it('should return a single result when there is only one relevant item', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Anastasia')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Anastasia');
+  });
+  it('should return multiple results when there are multiple relevant items', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Kari')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBeGreaterThanOrEqual(2);
+
+    result.current.data.forEach((item) => {
+      expect(item).toBeDefined();
+      expect(Object.values(item).values()).toContain('Kari');
+    });
+  });
+  it('should work with a partial query', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'anast')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Anastasia');
+  });
+  it('should work with nested field if not explicitly stated', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Titanic')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult.address).values()).toContain('Titanic');
+  });
+  it('should work with nested field if explicitly stated', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Titanic', {
+        indexing: {
+          fields: ['address.city'],
+        },
+      })
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult.address).values()).toContain('Titanic');
+  });
+  it('should work with nested ID', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Titanic', {
+        indexing: {
+          idKey: 'address.street',
+        },
+      })
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult.address).values()).toContain('Titanic');
+  });
+  it('should work with indexed Greek characters and Latin query', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Markos')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Μάρκος');
+  });
+  it('should work with indexed Greek characters and Greek query', async () => {
+    const { result, waitForNextUpdate } = renderHook(() =>
+      useSearch(documents, 'Μάρκος')
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Μάρκος');
+  });
+  it('should react to changes in the documents list', async () => {
+    const emptyDocuments: typeof documents = [];
+    const { result, rerender, waitForNextUpdate } = renderHook(
+      ({ documents }) => useSearch(documents, 'Anastasia'),
+      {
+        initialProps: { documents: emptyDocuments },
+      }
+    );
+    expect(result.current.loading).toBe(false);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual([]);
+
+    rerender({ documents });
+
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual([]);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual([]);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Anastasia');
+  });
+  it('should react to changes in the search query', async () => {
+    const { result, rerender, waitForNextUpdate } = renderHook(
+      ({ query }) => useSearch(documents, query),
+      {
+        initialProps: { query: '' },
+      }
+    );
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual(documents);
+
+    rerender({ query: 'Anastasia' });
+
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data).toEqual(documents);
+    act(() => result.current.search());
+    expect(result.current.loading).toBe(true);
+    expect(result.current.data).toEqual(documents);
+    await waitForNextUpdate();
+    expect(result.current.loading).toBe(false);
+    expect(result.current.data.length).toBe(1);
+
+    const firstResult = result.current.data[0] as NonNullable<
+      typeof result.current.data[0]
+    >;
+    expect(firstResult).toBeDefined();
+    expect(Object.values(firstResult).values()).toContain('Anastasia');
+  });
+});

package/dist/src/hook.ts

@@ -0,0 +1,50 @@
+import { useCallback, useMemo, useState } from 'react';
+import SearchIndex from './search';
+import { UseSearchOptions } from './types';
+
+/**
+ * Hook for searching through a list of documents
+ *
+ * Returns a list of documents that match the search query.
+ * If no query is provided, it returns the original list of documents.
+ *
+ * @param documents - The list of documents to search through
+ * @param options - Options for configuring the search index
+ * @param query - The search query
+ *
+ * @typeParam T - The type of the data in the documents list
+ */
+export default function useSearch<T extends Record<string, any>>(
+  documents: T[],
+  query?: string,
+  options?: UseSearchOptions<T>
+) {
+  const [loading, setLoading] = useState(false);
+  const [data, setData] = useState(documents);
+
+  const indexing = options?.indexing;
+
+  const index = useMemo(() => new SearchIndex(documents, indexing), [
+    documents,
+    indexing,
+  ]);
+
+  const search = useCallback(() => {
+    if (query) {
+      setLoading(true);
+      index
+        .searchAsync(documents, query)
+        .then((data) => setData(data))
+        .finally(() => setLoading(false));
+    } else {
+      setData(documents);
+    }
+  }, [query, index, documents]);
+
+  const reset = useCallback(() => {
+    setData(documents);
+    setLoading(false);
+  }, [documents]);
+
+  return { data, loading, search, reset };
+}

package/dist/src/index.ts

@@ -0,0 +1,4 @@
+import useSearch from './hook';
+
+export default useSearch;
+export * from './search';

package/dist/src/search/__tests__/utils.spec.ts

@@ -0,0 +1,73 @@
+import type { SimpleDocumentSearchResultSetUnit as ResultType } from 'flexsearch';
+import { findItemsByIds, getAllItemKeys, getResultIds } from '../utils';
+
+describe('utils', () => {
+  describe('getResultIds', () => {
+    it('should return the ids of the results without duplicates', () => {
+      const result: ResultType[] = [
+        { field: 'title', result: [1, 2] },
+        { field: 'description', result: [2, 3] },
+        { field: 'tag', result: [1, 2, 3, 4] },
+      ];
+      expect(getResultIds(result)).toEqual([1, 2, 3, 4]);
+    });
+    it('should return an empty array if there are no results', () => {
+      const result: ResultType[] = [];
+      expect(getResultIds(result)).toEqual([]);
+    });
+  });
+
+  describe('findItemsByIds', () => {
+    it('should return the items that match the given ids', () => {
+      const items = [
+        { id: 1, name: 'Anastasia' },
+        { id: 2, name: 'Bob' },
+        { id: 3, name: 'Cindy' },
+      ];
+      const ids = [1, 3];
+      expect(findItemsByIds(items, ids, 'id')).toEqual([
+        { id: 1, name: 'Anastasia' },
+        { id: 3, name: 'Cindy' },
+      ]);
+    });
+    it('should return an empty array if there are no items', () => {
+      const items: Record<string, any>[] = [];
+      const ids = [1, 3];
+      expect(findItemsByIds(items, ids, 'id')).toEqual([]);
+    });
+    it('should return an empty array if there are no ids', () => {
+      const items = [
+        { id: 1, name: 'Anastasia' },
+        { id: 2, name: 'Bob' },
+        { id: 3, name: 'Cindy' },
+      ];
+      const ids: number[] = [];
+      expect(findItemsByIds(items, ids, 'id')).toEqual([]);
+    });
+  });
+
+  describe('getAllItemKeys', () => {
+    it('should return all the keys for the given item', () => {
+      const item = {
+        id: 1,
+        name: 'Anastasia',
+        address: {
+          street: '123 Main St',
+          city: 'New York',
+          state: 'NY',
+        },
+      };
+      expect(getAllItemKeys(item)).toEqual([
+        'id',
+        'name',
+        'address.street',
+        'address.city',
+        'address.state',
+      ]);
+    });
+    it('should return an empty array if item is an empty object', () => {
+      const item: Record<string, any> = {};
+      expect(getAllItemKeys(item)).toEqual([]);
+    });
+  });
+});

package/dist/src/search/index.ts

@@ -0,0 +1,3 @@
+import { SearchIndex } from './search-index';
+
+export default SearchIndex;

package/dist/src/search/lang/gr/encoder.ts

@@ -0,0 +1,27 @@
+import { greekToGreeklishMap } from './normalization-map';
+
+const whitespaceRegex = /[\p{Z}\p{S}\p{P}\p{C}]+/u;
+const diacriticsRegex = /[\u0300-\u036f]/g;
+const greekCharsRegex = new RegExp(
+  Object.keys(greekToGreeklishMap).join('|'),
+  'igu'
+);
+
+/**
+ * Encode a string containing greek characters to greeklish
+ *
+ * This function is used to encode and tokenize a string containing greek characters.
+ * It replaces greek characters with their latin counterparts, while also removing
+ * any diacritics.
+ *
+ * @param str - The string to encode
+ * @returns An array of encoded words
+ */
+export function encodeGreek(str: string) {
+  return ('' + str)
+    .toLowerCase()
+    .normalize('NFD')
+    .replace(diacriticsRegex, '')
+    .replace(greekCharsRegex, (match) => (greekToGreeklishMap as any)[match])
+    .split(whitespaceRegex);
+}

package/dist/src/search/lang/gr/normalization-map.ts

@@ -0,0 +1,100 @@
+export const greekToGreeklishMap = {
+  ΓΧ: 'GX',
+  γχ: 'gx',
+  ΤΘ: 'T8',
+  τθ: 't8',
+  θη: '8h',
+  Θη: '8h',
+  ΘΗ: '8H',
+  αυ: 'au',
+  Αυ: 'Au',
+  ΑΥ: 'AY',
+  ευ: 'eu',
+  εύ: 'eu',
+  εϋ: 'ey',
+  εΰ: 'ey',
+  Ευ: 'Eu',
+  Εύ: 'Eu',
+  Εϋ: 'Ey',
+  Εΰ: 'Ey',
+  ΕΥ: 'EY',
+  ου: 'ou',
+  ού: 'ou',
+  οϋ: 'oy',
+  οΰ: 'oy',
+  Ου: 'Ou',
+  Ού: 'Ou',
+  Οϋ: 'Oy',
+  Οΰ: 'Oy',
+  ΟΥ: 'OY',
+  Α: 'A',
+  α: 'a',
+  ά: 'a',
+  Ά: 'A',
+  Β: 'B',
+  β: 'b',
+  Γ: 'G',
+  γ: 'g',
+  Δ: 'D',
+  δ: 'd',
+  Ε: 'E',
+  ε: 'e',
+  έ: 'e',
+  Έ: 'E',
+  Ζ: 'Z',
+  ζ: 'z',
+  Η: 'H',
+  η: 'h',
+  ή: 'h',
+  Ή: 'H',
+  Θ: 'TH',
+  θ: 'th',
+  Ι: 'I',
+  Ϊ: 'I',
+  ι: 'i',
+  ί: 'i',
+  ΐ: 'i',
+  ϊ: 'i',
+  Ί: 'I',
+  Κ: 'K',
+  κ: 'k',
+  Λ: 'L',
+  λ: 'l',
+  Μ: 'M',
+  μ: 'm',
+  Ν: 'N',
+  ν: 'n',
+  Ξ: 'KS',
+  ξ: 'ks',
+  Ο: 'O',
+  ο: 'o',
+  Ό: 'O',
+  ό: 'o',
+  Π: 'P',
+  π: 'p',
+  Ρ: 'R',
+  ρ: 'r',
+  Σ: 'S',
+  σ: 's',
+  Τ: 'T',
+  τ: 't',
+  Υ: 'Y',
+  Ύ: 'Y',
+  Ϋ: 'Y',
+  ΰ: 'y',
+  ύ: 'y',
+  ϋ: 'y',
+  υ: 'y',
+  Φ: 'F',
+  φ: 'f',
+  Χ: 'X',
+  χ: 'x',
+  Ψ: 'Ps',
+  ψ: 'ps',
+  Ω: 'w',
+  ω: 'w',
+  Ώ: 'w',
+  ώ: 'w',
+  ς: 's',
+  ';': '?',
+};

package/dist/src/search/search-index.ts

@@ -0,0 +1,103 @@
+import { Document } from 'flexsearch';
+import type { SearchIndexOptions } from '../types';
+import { encodeGreek } from './lang/gr/encoder';
+import { findItemsByIds, getAllItemKeys, getResultIds } from './utils';
+
+/**
+ * Wrapper class around third party index and search library
+ *
+ * @typeParam T - The type of the data that will be indexed
+ */
+export class SearchIndex<T extends Record<string, any>> {
+  /**
+   * Instance of the third party library's index
+   */
+  private index: Document<T>;
+  private idKey: string;
+
+  constructor(items: T[], options?: SearchIndexOptions<T>) {
+    this.idKey = 'id';
+    if (options && options.idKey) {
+      this.idKey = options.idKey.replace('.', ':');
+    } else if (!(items[0] && 'id' in items[0])) {
+      items = items.map((item, index) => ({ ...item, id: index }));
+    }
+
+    let fields: string[] = [];
+    if (options && options.fields) {
+      fields = options.fields;
+    } else if (items[0]) {
+      fields = getAllItemKeys(items[0]);
+    }
+    fields = fields
+      .filter((field) => field !== this.idKey)
+      .map((field) => field.replace('.', ':'));
+
+    this.index = new Document<T>({
+      document: {
+        id: this.idKey,
+        index: fields,
+      },
+      tokenize: 'forward',
+      worker: options?.enableWorker,
+      encode: encodeGreek,
+    });
+
+    this.addAll(items);
+  }
+
+  /**
+   * Add an item to the index
+   *
+   * @param item - The item to add to the index
+   */
+  public add(item: T) {
+    this.index.add(item);
+    return this;
+  }
+
+  /**
+   * Add multiple items to the index
+   *
+   * @param items - The list of items to add to the index
+   */
+  public addAll(items: T[]) {
+    items.forEach((item) => this.index.add(item));
+    return this;
+  }
+
+  /**
+   * Search the index for the given term
+   *
+   * @param documents - The list of documents to search
+   * @param searchTerm - The term to search for
+   * @returns The list of items that match the search term
+   */
+  public search(documents: T[], searchTerm: string) {
+    const res = this.index.search(searchTerm);
+    const ids = getResultIds(res);
+    return findItemsByIds(
+      documents,
+      ids,
+      this.idKey as Extract<keyof T, string>
+    );
+  }
+
+  /**
+   * Asynchronously search the index for the given term
+   *
+   * @param documents - The list of documents to search
+   * @param searchTerm - The term to search for
+   * @returns The list of items that match the search term
+   */
+  public async searchAsync(documents: T[], searchTerm: string) {
+    return this.index
+      .searchAsync(searchTerm)
+      .then((res) => {
+        return getResultIds(res);
+      })
+      .then((res) =>
+        findItemsByIds(documents, res, this.idKey as Extract<keyof T, string>)
+      );
+  }
+}