@tinacms/search 0.0.0-bf8b9b7-20251204000148 → 0.0.0-c19d29e-20251224001156

package/README.md

@@ -0,0 +1,55 @@
+# @tinacms/search
+
+Full-text search for TinaCMS powered by [search-index](https://github.com/fergiemcdowall/search-index).
+
+## Installation
+
+```bash
+pnpm add @tinacms/search
+```
+
+## Usage
+
+```typescript
+import { LocalSearchIndexClient } from "@tinacms/search";
+
+const client = new LocalSearchIndexClient({
+  stopwordLanguages: ["eng"],
+});
+
+await client.onStartIndexing();
+
+await client.put([
+  {
+    _id: "1",
+    title: "Getting Started",
+    body: "TinaCMS is a Git-backed headless CMS",
+  },
+  {
+    _id: "2",
+    title: "React Tutorial",
+    body: "Learn how to build React applications",
+  },
+]);
+
+// Basic search
+const results = await client.query("TinaCMS", { limit: 10 });
+
+// Fuzzy search (handles typos)
+const fuzzyResults = await client.query("TinCMS tutrial", {
+  fuzzy: true,
+  limit: 10,
+});
+```
+
+## API
+
+- `client.onStartIndexing()` - Initialize the index
+- `client.put(documents)` - Index documents
+- `client.query(query, options)` - Search the index
+- `client.del(ids)` - Delete documents
+- `client.export(filename)` - Export index to SQLite
+
+## License
+
+Apache 2.0

package/dist/client/index.d.ts

@@ -1,5 +1,6 @@
-import type { SearchClient } from '../types';
+import type { SearchClient, SearchOptions, SearchQueryResponse, IndexableDocument, SearchIndex } from '../types';
 import { MemoryLevel } from 'memory-level';
+import { FuzzySearchWrapper } from '../fuzzy-search-wrapper';
 type TinaSearchIndexerClientOptions = {
     stopwordLanguages?: string[];
     tokenSplitRegex?: string;
@@ -10,23 +11,16 @@ type TinaCloudSearchIndexerClientOptions = {
     indexerToken: string;
 } & TinaSearchIndexerClientOptions;
 export declare class LocalSearchIndexClient implements SearchClient {
-    searchIndex: any;
+    searchIndex?: SearchIndex;
     protected readonly memoryLevel: MemoryLevel;
     private readonly stopwords;
     private readonly tokenSplitRegex;
+    fuzzySearchWrapper?: FuzzySearchWrapper;
     constructor(options: TinaSearchIndexerClientOptions);
     onStartIndexing(): Promise<void>;
-    put(docs: any[]): Promise<any>;
-    del(ids: string[]): Promise<any>;
-    query(query: string, options: {
-        cursor?: string;
-        limit?: number;
-    } | undefined): Promise<{
-        results: any[];
-        total: number;
-        nextCursor: string | null;
-        prevCursor: string | null;
-    }>;
+    put(docs: IndexableDocument[]): Promise<void>;
+    del(ids: string[]): Promise<void>;
+    query(query: string, options?: SearchOptions): Promise<SearchQueryResponse>;
     export(filename: string): Promise<void>;
 }
 export declare class TinaCMSSearchIndexClient extends LocalSearchIndexClient {
@@ -34,6 +28,9 @@ export declare class TinaCMSSearchIndexClient extends LocalSearchIndexClient {
     private readonly branch;
     private readonly indexerToken;
     constructor(options: TinaCloudSearchIndexerClientOptions);
+    private getUploadUrl;
+    private serializeIndex;
+    private uploadIndex;
     onFinishIndexing(): Promise<void>;
 }
 export {};

package/dist/fuzzy/cache.d.ts

@@ -0,0 +1,11 @@
+import type { FuzzyMatch, FuzzySearchOptions } from './types';
+export declare class FuzzyCache {
+    private cache;
+    private maxSize;
+    constructor(maxSize?: number);
+    private getCacheKey;
+    get(query: string, options: Partial<FuzzySearchOptions>): FuzzyMatch[] | undefined;
+    set(query: string, options: Partial<FuzzySearchOptions>, results: FuzzyMatch[]): void;
+    clear(): void;
+    get size(): number;
+}

package/dist/fuzzy/distance.d.ts

@@ -0,0 +1,8 @@
+import type { FuzzySearchOptions, FuzzyMatch } from './types';
+export declare const PREFIX_MATCH_MIN_SIMILARITY = 0.8;
+export declare function levenshteinDistance(str1: string, str2: string): number;
+export declare function similarityScore(str1: string, str2: string, useTranspositions?: boolean): number;
+export declare function damerauLevenshteinDistance(str1: string, str2: string): number;
+export declare function getNgrams(str: string, n?: number): Set<string>;
+export declare function ngramOverlap(ngrams1: Set<string>, ngrams2: Set<string>): number;
+export declare function findSimilarTerms(query: string, dictionary: string[], options?: FuzzySearchOptions): FuzzyMatch[];

package/dist/fuzzy/index.d.ts

@@ -0,0 +1,4 @@
+export type { FuzzySearchOptions, FuzzyMatch } from './types';
+export { DEFAULT_FUZZY_OPTIONS } from './types';
+export { FuzzyCache } from './cache';
+export { levenshteinDistance, similarityScore, damerauLevenshteinDistance, findSimilarTerms, getNgrams, ngramOverlap, PREFIX_MATCH_MIN_SIMILARITY, } from './distance';

package/dist/fuzzy/types.d.ts

@@ -0,0 +1,19 @@
+export interface FuzzySearchOptions {
+    maxDistance?: number;
+    minSimilarity?: number;
+    maxResults?: number;
+    useTranspositions?: boolean;
+    caseSensitive?: boolean;
+    /** Use n-gram filtering for candidate selection (supports transpositions) */
+    useNgramFilter?: boolean;
+    /** Size of n-grams for filtering (default: 2) */
+    ngramSize?: number;
+    /** Minimum n-gram overlap ratio to consider a candidate (0-1, default: 0.2) */
+    minNgramOverlap?: number;
+}
+export interface FuzzyMatch {
+    term: string;
+    distance: number;
+    similarity: number;
+}
+export declare const DEFAULT_FUZZY_OPTIONS: Required<FuzzySearchOptions>;

package/dist/fuzzy-search-wrapper.d.ts

@@ -0,0 +1,23 @@
+import type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
+import type { SearchQueryResponse, SearchIndex } from './types';
+interface QueryOptions {
+    limit?: number;
+    cursor?: string;
+    fuzzyOptions?: FuzzySearchOptions;
+}
+export declare class FuzzySearchWrapper {
+    private cache;
+    private searchIndex;
+    constructor(searchIndex: SearchIndex, cacheSize?: number);
+    getDictionary(field?: string): Promise<string[]>;
+    findSimilar(query: string, field?: string, options?: FuzzySearchOptions): Promise<FuzzyMatch[]>;
+    expandQuery(query: string, options?: FuzzySearchOptions): Promise<{
+        original: string[];
+        expanded: string[];
+        matches: Record<string, FuzzyMatch[]>;
+    }>;
+    query(query: string, options?: QueryOptions): Promise<SearchQueryResponse>;
+    clearCache(): void;
+    getCacheSize(): number;
+}
+export {};

package/dist/index-client.d.ts

@@ -1,18 +1,28 @@
-export type { SearchClient } from './types';
+export type { SearchClient, SearchResult, SearchQueryResponse, IndexableDocument, SearchOptions, } from './types';
+export type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
 export { processDocumentForIndexing } from './indexer/utils';
-export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?: string[]) => any;
-export declare const optionsToSearchIndexOptions: (options?: {
+import type { FuzzyMatch } from './fuzzy';
+import type { SearchResult, SearchQueryResponse } from './types';
+interface SearchQuery {
+    AND: string[];
+}
+interface PaginationOptions {
     limit?: number;
     cursor?: string;
-}) => {
-    PAGE?: {};
-};
-export declare const parseSearchIndexResponse: (data: any, options?: {
-    limit?: number;
-    cursor?: string;
-}) => {
-    results: any;
-    total: any;
-    prevCursor: any;
-    nextCursor: string;
-};
+}
+interface PageOptions {
+    PAGE?: {
+        SIZE: number;
+        NUMBER: number;
+    };
+}
+interface SearchIndexResponse {
+    RESULT: SearchResult[];
+    RESULT_LENGTH: number;
+    FUZZY_MATCHES?: Record<string, FuzzyMatch[]>;
+    NEXT_CURSOR?: string | null;
+    PREV_CURSOR?: string | null;
+}
+export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?: string[]) => SearchQuery;
+export declare const optionsToSearchIndexOptions: (options?: PaginationOptions) => PageOptions;
+export declare const parseSearchIndexResponse: (data: SearchIndexResponse, options?: PaginationOptions) => SearchQueryResponse;

package/dist/index-client.js

@@ -5,195 +5,202 @@ var __publicField = (obj, key, value) => {
   return value;
 };
 import * as sw from "stopword";
+const INDEXABLE_NODE_TYPES = ["text", "code_block", "html"];
 class StringBuilder {
   constructor(limit) {
-    __publicField(this, "buffer");
-    __publicField(this, "length", 0);
+    __publicField(this, "buffer", []);
     __publicField(this, "limit");
-    this.buffer = [];
+    __publicField(this, "length", 0);
     this.limit = limit;
   }
   append(str) {
-    if (this.length + str.length > this.limit) {
+    if (this.length + str.length > this.limit)
       return true;
-    } else {
-      this.buffer.push(str);
-      this.length += str.length;
-      if (this.length > this.limit) {
-        return true;
-      }
-      return false;
-    }
+    this.buffer.push(str);
+    this.length += str.length;
+    return this.length > this.limit;
   }
   toString() {
     return this.buffer.join(" ");
   }
 }
-const extractText = (data, acc, indexableNodeTypes) => {
-  var _a, _b;
-  if (data) {
-    if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
-      const tokens = tokenizeString(data.text || data.value);
-      for (const token of tokens) {
-        if (acc.append(token)) {
-          return;
-        }
-      }
+const tokenizeString = (str) => {
+  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+};
+const extractText = (data, builder, nodeTypes) => {
+  var _a;
+  if (!data)
+    return;
+  if (nodeTypes.includes(data.type ?? "") && (data.text || data.value)) {
+    const tokens = tokenizeString(data.text || data.value || "");
+    for (const token of tokens) {
+      if (builder.append(token))
+        return;
     }
-    (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
-      _a,
-      (child) => extractText(child, acc, indexableNodeTypes)
-    );
   }
+  (_a = data.children) == null ? void 0 : _a.forEach((child) => extractText(child, builder, nodeTypes));
 };
-const relativePath = (path, collection) => {
+const getRelativePath = (path, collection) => {
   return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
 };
-const tokenizeString = (str) => {
-  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
-};
-const processTextFieldValue = (value, maxLen) => {
+const processTextField = (value, maxLength) => {
   const tokens = tokenizeString(value);
-  const builder = new StringBuilder(maxLen);
+  const builder = new StringBuilder(maxLength);
   for (const part of tokens) {
-    if (builder.append(part)) {
+    if (builder.append(part))
       break;
-    }
   }
   return builder.toString();
 };
+const processRichTextField = (value, maxLength) => {
+  const builder = new StringBuilder(maxLength);
+  extractText(value, builder, INDEXABLE_NODE_TYPES);
+  return builder.toString();
+};
+const processObjectField = (data, path, collection, textIndexLength, field) => {
+  if (field.list) {
+    return data.map(
+      (obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, field)
+    );
+  }
+  return processDocumentForIndexing(
+    data,
+    path,
+    collection,
+    textIndexLength,
+    field
+  );
+};
+const processStringField = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processTextField(value, maxLength)
+    );
+  }
+  return processTextField(data, maxLength);
+};
+const processRichTextFieldData = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processRichTextField(value, maxLength)
+    );
+  }
+  return processRichTextField(data, maxLength);
+};
 const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
   if (!field) {
-    const relPath = relativePath(path, collection);
-    data["_id"] = `${collection.name}:${relPath}`;
-    data["_relativePath"] = relPath;
+    const relativePath = getRelativePath(path, collection);
+    data["_id"] = `${collection.name}:${relativePath}`;
+    data["_relativePath"] = relativePath;
   }
-  for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
+  const fields = (field == null ? void 0 : field.fields) || collection.fields || [];
+  for (const f of fields) {
     if (!f.searchable) {
       delete data[f.name];
       continue;
     }
-    const isList = f.list;
-    if (data[f.name]) {
-      if (f.type === "object") {
-        if (isList) {
-          data[f.name] = data[f.name].map(
-            (obj) => processDocumentForIndexing(
-              obj,
-              path,
-              collection,
-              textIndexLength,
-              f
-            )
-          );
-        } else {
-          data[f.name] = processDocumentForIndexing(
-            data[f.name],
-            path,
-            collection,
-            textIndexLength,
-            f
-          );
-        }
-      } else if (f.type === "string") {
-        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
-        if (isList) {
-          data[f.name] = data[f.name].map(
-            (value) => processTextFieldValue(value, fieldTextIndexLength)
-          );
-        } else {
-          data[f.name] = processTextFieldValue(
-            data[f.name],
-            fieldTextIndexLength
-          );
-        }
-      } else if (f.type === "rich-text") {
-        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
-        if (isList) {
-          data[f.name] = data[f.name].map((value) => {
-            const acc = new StringBuilder(fieldTextIndexLength);
-            extractText(value, acc, ["text", "code_block", "html"]);
-            return acc.toString();
-          });
-        } else {
-          const acc = new StringBuilder(fieldTextIndexLength);
-          extractText(data[f.name], acc, ["text", "code_block", "html"]);
-          data[f.name] = acc.toString();
-        }
-      }
+    if (!data[f.name])
+      continue;
+    const fieldMaxLength = f.maxSearchIndexFieldLength || textIndexLength;
+    const isList = Boolean(f.list);
+    switch (f.type) {
+      case "object":
+        data[f.name] = processObjectField(
+          data[f.name],
+          path,
+          collection,
+          textIndexLength,
+          f
+        );
+        break;
+      case "string":
+        data[f.name] = processStringField(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
+        break;
+      case "rich-text":
+        data[f.name] = processRichTextFieldData(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
+        break;
     }
   }
   return data;
 };
-const memo = {};
+const stopwordCache = {};
 const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
-  let stopwords = defaultStopWords;
-  if (keys) {
-    if (memo[keys.join(",")]) {
-      return memo[keys.join(",")];
-    }
-    stopwords = [];
-    for (const key of keys) {
-      stopwords.push(...sw[key]);
-    }
-    memo[keys.join(",")] = stopwords;
+  if (!keys) {
+    return defaultStopWords;
   }
+  const cacheKey = keys.join(",");
+  if (stopwordCache[cacheKey]) {
+    return stopwordCache[cacheKey];
+  }
+  const stopwords = keys.flatMap((key) => sw[key] || []);
+  stopwordCache[cacheKey] = stopwords;
   return stopwords;
 };
 const queryToSearchIndexQuery = (query, stopwordLanguages) => {
-  let q;
   const parts = query.split(" ");
   const stopwords = lookupStopwords(stopwordLanguages);
   if (parts.length === 1) {
-    q = { AND: [parts[0]] };
-  } else {
-    q = {
-      AND: parts.filter(
-        (part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
-      )
-    };
+    return { AND: [parts[0]] };
   }
-  return q;
+  const filteredParts = parts.filter(
+    (part) => part.toLowerCase() !== "and" && !stopwords.includes(part.toLowerCase())
+  );
+  return { AND: filteredParts };
 };
 const optionsToSearchIndexOptions = (options) => {
-  const opt = {};
-  if (options == null ? void 0 : options.limit) {
-    opt["PAGE"] = {
+  if (!(options == null ? void 0 : options.limit))
+    return {};
+  return {
+    PAGE: {
       SIZE: options.limit,
-      NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
-    };
-  }
-  return opt;
+      NUMBER: options.cursor ? parseInt(options.cursor) : 0
+    }
+  };
 };
 const parseSearchIndexResponse = (data, options) => {
-  const results = data["RESULT"];
-  const total = data["RESULT_LENGTH"];
-  if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
-    const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
-    const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
+  const resultArray = (data == null ? void 0 : data.RESULT) ?? (data == null ? void 0 : data.results);
+  if (!data || !Array.isArray(resultArray)) {
     return {
-      results,
-      total,
-      prevCursor,
-      nextCursor
-    };
-  } else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
-    const prevCursor = null;
-    const nextCursor = total <= options.limit ? null : "1";
-    return {
-      results,
-      total,
-      prevCursor,
-      nextCursor
+      results: [],
+      total: 0,
+      prevCursor: null,
+      nextCursor: null,
+      fuzzyMatches: void 0
     };
-  } else {
+  }
+  const results = data.RESULT ?? data.results;
+  const total = data.RESULT_LENGTH ?? data.total ?? 0;
+  const fuzzyMatches = data.FUZZY_MATCHES ?? data.fuzzyMatches;
+  const nextCursor = data.NEXT_CURSOR ?? data.nextCursor;
+  const prevCursor = data.PREV_CURSOR ?? data.prevCursor;
+  if (nextCursor !== void 0 || prevCursor !== void 0) {
     return {
       results,
       total,
-      prevCursor: null,
-      nextCursor: null
+      prevCursor: prevCursor ?? null,
+      nextCursor: nextCursor ?? null,
+      fuzzyMatches
     };
   }
+  const currentPage = (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0;
+  const pageSize = options == null ? void 0 : options.limit;
+  const hasPreviousPage = currentPage > 0;
+  const hasNextPage = pageSize ? total > (currentPage + 1) * pageSize : false;
+  return {
+    results,
+    total,
+    prevCursor: hasPreviousPage ? (currentPage - 1).toString() : null,
+    nextCursor: hasNextPage ? (currentPage + 1).toString() : null,
+    fuzzyMatches
+  };
 };
 export {
   optionsToSearchIndexOptions,

package/dist/index.d.ts

@@ -1,5 +1,10 @@
-import si from 'search-index';
+import createSearchIndex from 'search-index';
 export { SearchIndexer } from './indexer';
 export { LocalSearchIndexClient, TinaCMSSearchIndexClient } from './client';
-export type { SearchClient } from './types';
-export { si };
+export type { SearchClient, SearchOptions, SearchResult, SearchQueryResponse, IndexableDocument, SearchIndexResult, SearchIndex, } from './types';
+export type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
+export { levenshteinDistance, similarityScore, damerauLevenshteinDistance, findSimilarTerms, FuzzyCache, DEFAULT_FUZZY_OPTIONS, } from './fuzzy';
+export { FuzzySearchWrapper } from './fuzzy-search-wrapper';
+export { buildPageOptions, buildPaginationCursors } from './pagination';
+export type { PaginationOptions, PageOptions, PaginationCursors, } from './pagination';
+export { createSearchIndex };