@tinacms/search 0.0.0-c1132cd-20241024060747 → 0.0.0-c19d29e-20251224001156

package/README.md

@@ -0,0 +1,55 @@
+# @tinacms/search
+
+Full-text search for TinaCMS powered by [search-index](https://github.com/fergiemcdowall/search-index).
+
+## Installation
+
+```bash
+pnpm add @tinacms/search
+```
+
+## Usage
+
+```typescript
+import { LocalSearchIndexClient } from "@tinacms/search";
+
+const client = new LocalSearchIndexClient({
+  stopwordLanguages: ["eng"],
+});
+
+await client.onStartIndexing();
+
+await client.put([
+  {
+    _id: "1",
+    title: "Getting Started",
+    body: "TinaCMS is a Git-backed headless CMS",
+  },
+  {
+    _id: "2",
+    title: "React Tutorial",
+    body: "Learn how to build React applications",
+  },
+]);
+
+// Basic search
+const results = await client.query("TinaCMS", { limit: 10 });
+
+// Fuzzy search (handles typos)
+const fuzzyResults = await client.query("TinCMS tutrial", {
+  fuzzy: true,
+  limit: 10,
+});
+```
+
+## API
+
+- `client.onStartIndexing()` - Initialize the index
+- `client.put(documents)` - Index documents
+- `client.query(query, options)` - Search the index
+- `client.del(ids)` - Delete documents
+- `client.export(filename)` - Export index to SQLite
+
+## License
+
+Apache 2.0

package/dist/client/index.d.ts

@@ -1,5 +1,6 @@
-import type { SearchClient } from '../types';
+import type { SearchClient, SearchOptions, SearchQueryResponse, IndexableDocument, SearchIndex } from '../types';
 import { MemoryLevel } from 'memory-level';
+import { FuzzySearchWrapper } from '../fuzzy-search-wrapper';
 type TinaSearchIndexerClientOptions = {
     stopwordLanguages?: string[];
     tokenSplitRegex?: string;
@@ -10,23 +11,16 @@ type TinaCloudSearchIndexerClientOptions = {
     indexerToken: string;
 } & TinaSearchIndexerClientOptions;
 export declare class LocalSearchIndexClient implements SearchClient {
-    searchIndex: any;
+    searchIndex?: SearchIndex;
     protected readonly memoryLevel: MemoryLevel;
     private readonly stopwords;
     private readonly tokenSplitRegex;
+    fuzzySearchWrapper?: FuzzySearchWrapper;
     constructor(options: TinaSearchIndexerClientOptions);
     onStartIndexing(): Promise<void>;
-    put(docs: any[]): Promise<any>;
-    del(ids: string[]): Promise<any>;
-    query(query: string, options: {
-        cursor?: string;
-        limit?: number;
-    } | undefined): Promise<{
-        results: any[];
-        total: number;
-        nextCursor: string | null;
-        prevCursor: string | null;
-    }>;
+    put(docs: IndexableDocument[]): Promise<void>;
+    del(ids: string[]): Promise<void>;
+    query(query: string, options?: SearchOptions): Promise<SearchQueryResponse>;
     export(filename: string): Promise<void>;
 }
 export declare class TinaCMSSearchIndexClient extends LocalSearchIndexClient {
@@ -34,6 +28,9 @@ export declare class TinaCMSSearchIndexClient extends LocalSearchIndexClient {
     private readonly branch;
     private readonly indexerToken;
     constructor(options: TinaCloudSearchIndexerClientOptions);
+    private getUploadUrl;
+    private serializeIndex;
+    private uploadIndex;
     onFinishIndexing(): Promise<void>;
 }
 export {};

package/dist/fuzzy/cache.d.ts

@@ -0,0 +1,11 @@
+import type { FuzzyMatch, FuzzySearchOptions } from './types';
+export declare class FuzzyCache {
+    private cache;
+    private maxSize;
+    constructor(maxSize?: number);
+    private getCacheKey;
+    get(query: string, options: Partial<FuzzySearchOptions>): FuzzyMatch[] | undefined;
+    set(query: string, options: Partial<FuzzySearchOptions>, results: FuzzyMatch[]): void;
+    clear(): void;
+    get size(): number;
+}

package/dist/fuzzy/distance.d.ts

@@ -0,0 +1,8 @@
+import type { FuzzySearchOptions, FuzzyMatch } from './types';
+export declare const PREFIX_MATCH_MIN_SIMILARITY = 0.8;
+export declare function levenshteinDistance(str1: string, str2: string): number;
+export declare function similarityScore(str1: string, str2: string, useTranspositions?: boolean): number;
+export declare function damerauLevenshteinDistance(str1: string, str2: string): number;
+export declare function getNgrams(str: string, n?: number): Set<string>;
+export declare function ngramOverlap(ngrams1: Set<string>, ngrams2: Set<string>): number;
+export declare function findSimilarTerms(query: string, dictionary: string[], options?: FuzzySearchOptions): FuzzyMatch[];

package/dist/fuzzy/index.d.ts

@@ -0,0 +1,4 @@
+export type { FuzzySearchOptions, FuzzyMatch } from './types';
+export { DEFAULT_FUZZY_OPTIONS } from './types';
+export { FuzzyCache } from './cache';
+export { levenshteinDistance, similarityScore, damerauLevenshteinDistance, findSimilarTerms, getNgrams, ngramOverlap, PREFIX_MATCH_MIN_SIMILARITY, } from './distance';

package/dist/fuzzy/types.d.ts

@@ -0,0 +1,19 @@
+export interface FuzzySearchOptions {
+    maxDistance?: number;
+    minSimilarity?: number;
+    maxResults?: number;
+    useTranspositions?: boolean;
+    caseSensitive?: boolean;
+    /** Use n-gram filtering for candidate selection (supports transpositions) */
+    useNgramFilter?: boolean;
+    /** Size of n-grams for filtering (default: 2) */
+    ngramSize?: number;
+    /** Minimum n-gram overlap ratio to consider a candidate (0-1, default: 0.2) */
+    minNgramOverlap?: number;
+}
+export interface FuzzyMatch {
+    term: string;
+    distance: number;
+    similarity: number;
+}
+export declare const DEFAULT_FUZZY_OPTIONS: Required<FuzzySearchOptions>;

package/dist/fuzzy-search-wrapper.d.ts

@@ -0,0 +1,23 @@
+import type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
+import type { SearchQueryResponse, SearchIndex } from './types';
+interface QueryOptions {
+    limit?: number;
+    cursor?: string;
+    fuzzyOptions?: FuzzySearchOptions;
+}
+export declare class FuzzySearchWrapper {
+    private cache;
+    private searchIndex;
+    constructor(searchIndex: SearchIndex, cacheSize?: number);
+    getDictionary(field?: string): Promise<string[]>;
+    findSimilar(query: string, field?: string, options?: FuzzySearchOptions): Promise<FuzzyMatch[]>;
+    expandQuery(query: string, options?: FuzzySearchOptions): Promise<{
+        original: string[];
+        expanded: string[];
+        matches: Record<string, FuzzyMatch[]>;
+    }>;
+    query(query: string, options?: QueryOptions): Promise<SearchQueryResponse>;
+    clearCache(): void;
+    getCacheSize(): number;
+}
+export {};

package/dist/index-client.d.ts

@@ -1,16 +1,28 @@
-export type { SearchClient } from './types';
+export type { SearchClient, SearchResult, SearchQueryResponse, IndexableDocument, SearchOptions, } from './types';
+export type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
 export { processDocumentForIndexing } from './indexer/utils';
-export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?: string[]) => any;
-export declare const optionsToSearchIndexOptions: (options?: {
+import type { FuzzyMatch } from './fuzzy';
+import type { SearchResult, SearchQueryResponse } from './types';
+interface SearchQuery {
+    AND: string[];
+}
+interface PaginationOptions {
     limit?: number;
     cursor?: string;
-}) => {};
-export declare const parseSearchIndexResponse: (data: any, options?: {
-    limit?: number;
-    cursor?: string;
-}) => {
-    results: any;
-    total: any;
-    prevCursor: any;
-    nextCursor: string;
-};
+}
+interface PageOptions {
+    PAGE?: {
+        SIZE: number;
+        NUMBER: number;
+    };
+}
+interface SearchIndexResponse {
+    RESULT: SearchResult[];
+    RESULT_LENGTH: number;
+    FUZZY_MATCHES?: Record<string, FuzzyMatch[]>;
+    NEXT_CURSOR?: string | null;
+    PREV_CURSOR?: string | null;
+}
+export declare const queryToSearchIndexQuery: (query: string, stopwordLanguages?: string[]) => SearchQuery;
+export declare const optionsToSearchIndexOptions: (options?: PaginationOptions) => PageOptions;
+export declare const parseSearchIndexResponse: (data: SearchIndexResponse, options?: PaginationOptions) => SearchQueryResponse;

package/dist/index-client.js

@@ -1,215 +1,210 @@
-(function(global, factory) {
-  typeof exports === "object" && typeof module !== "undefined" ? factory(exports, require("stopword")) : typeof define === "function" && define.amd ? define(["exports", "stopword"], factory) : (global = typeof globalThis !== "undefined" ? globalThis : global || self, factory(global["@tinacms/search"] = {}, global.NOOP));
-})(this, function(exports2, sw) {
-  "use strict";
-  function _interopNamespaceDefault(e) {
-    const n = Object.create(null, { [Symbol.toStringTag]: { value: "Module" } });
-    if (e) {
-      for (const k in e) {
-        if (k !== "default") {
-          const d = Object.getOwnPropertyDescriptor(e, k);
-          Object.defineProperty(n, k, d.get ? d : {
-            enumerable: true,
-            get: () => e[k]
-          });
-        }
-      }
-    }
-    n.default = e;
-    return Object.freeze(n);
+var __defProp = Object.defineProperty;
+var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
+var __publicField = (obj, key, value) => {
+  __defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
+  return value;
+};
+import * as sw from "stopword";
+const INDEXABLE_NODE_TYPES = ["text", "code_block", "html"];
+class StringBuilder {
+  constructor(limit) {
+    __publicField(this, "buffer", []);
+    __publicField(this, "limit");
+    __publicField(this, "length", 0);
+    this.limit = limit;
   }
-  const sw__namespace = /* @__PURE__ */ _interopNamespaceDefault(sw);
-  class StringBuilder {
-    constructor(limit) {
-      this.length = 0;
-      this.buffer = [];
-      this.limit = limit;
-    }
-    append(str) {
-      if (this.length + str.length > this.limit) {
-        return true;
-      } else {
-        this.buffer.push(str);
-        this.length += str.length;
-        if (this.length > this.limit) {
-          return true;
-        }
-        return false;
-      }
-    }
-    toString() {
-      return this.buffer.join(" ");
+  append(str) {
+    if (this.length + str.length > this.limit)
+      return true;
+    this.buffer.push(str);
+    this.length += str.length;
+    return this.length > this.limit;
+  }
+  toString() {
+    return this.buffer.join(" ");
+  }
+}
+const tokenizeString = (str) => {
+  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+};
+const extractText = (data, builder, nodeTypes) => {
+  var _a;
+  if (!data)
+    return;
+  if (nodeTypes.includes(data.type ?? "") && (data.text || data.value)) {
+    const tokens = tokenizeString(data.text || data.value || "");
+    for (const token of tokens) {
+      if (builder.append(token))
+        return;
     }
   }
-  const extractText = (data, acc, indexableNodeTypes) => {
-    var _a, _b;
-    if (data) {
-      if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
-        const tokens = tokenizeString(data.text || data.value);
-        for (const token of tokens) {
-          if (acc.append(token)) {
-            return;
-          }
-        }
-      }
-      (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
-        _a,
-        (child) => extractText(child, acc, indexableNodeTypes)
-      );
+  (_a = data.children) == null ? void 0 : _a.forEach((child) => extractText(child, builder, nodeTypes));
+};
+const getRelativePath = (path, collection) => {
+  return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
+};
+const processTextField = (value, maxLength) => {
+  const tokens = tokenizeString(value);
+  const builder = new StringBuilder(maxLength);
+  for (const part of tokens) {
+    if (builder.append(part))
+      break;
+  }
+  return builder.toString();
+};
+const processRichTextField = (value, maxLength) => {
+  const builder = new StringBuilder(maxLength);
+  extractText(value, builder, INDEXABLE_NODE_TYPES);
+  return builder.toString();
+};
+const processObjectField = (data, path, collection, textIndexLength, field) => {
+  if (field.list) {
+    return data.map(
+      (obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, field)
+    );
+  }
+  return processDocumentForIndexing(
+    data,
+    path,
+    collection,
+    textIndexLength,
+    field
+  );
+};
+const processStringField = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processTextField(value, maxLength)
+    );
+  }
+  return processTextField(data, maxLength);
+};
+const processRichTextFieldData = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processRichTextField(value, maxLength)
+    );
+  }
+  return processRichTextField(data, maxLength);
+};
+const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
+  if (!field) {
+    const relativePath = getRelativePath(path, collection);
+    data["_id"] = `${collection.name}:${relativePath}`;
+    data["_relativePath"] = relativePath;
+  }
+  const fields = (field == null ? void 0 : field.fields) || collection.fields || [];
+  for (const f of fields) {
+    if (!f.searchable) {
+      delete data[f.name];
+      continue;
     }
-  };
-  const relativePath = (path, collection) => {
-    return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
-  };
-  const tokenizeString = (str) => {
-    return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
-  };
-  const processTextFieldValue = (value, maxLen) => {
-    const tokens = tokenizeString(value);
-    const builder = new StringBuilder(maxLen);
-    for (const part of tokens) {
-      if (builder.append(part)) {
+    if (!data[f.name])
+      continue;
+    const fieldMaxLength = f.maxSearchIndexFieldLength || textIndexLength;
+    const isList = Boolean(f.list);
+    switch (f.type) {
+      case "object":
+        data[f.name] = processObjectField(
+          data[f.name],
+          path,
+          collection,
+          textIndexLength,
+          f
+        );
+        break;
+      case "string":
+        data[f.name] = processStringField(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
+        break;
+      case "rich-text":
+        data[f.name] = processRichTextFieldData(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
         break;
-      }
-    }
-    return builder.toString();
-  };
-  const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
-    if (!field) {
-      const relPath = relativePath(path, collection);
-      data["_id"] = `${collection.name}:${relPath}`;
-      data["_relativePath"] = relPath;
-    }
-    for (const f of (field == null ? void 0 : field.fields) || collection.fields || []) {
-      if (!f.searchable) {
-        delete data[f.name];
-        continue;
-      }
-      const isList = f.list;
-      if (data[f.name]) {
-        if (f.type === "object") {
-          if (isList) {
-            data[f.name] = data[f.name].map(
-              (obj) => processDocumentForIndexing(
-                obj,
-                path,
-                collection,
-                textIndexLength,
-                f
-              )
-            );
-          } else {
-            data[f.name] = processDocumentForIndexing(
-              data[f.name],
-              path,
-              collection,
-              textIndexLength,
-              f
-            );
-          }
-        } else if (f.type === "string") {
-          const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
-          if (isList) {
-            data[f.name] = data[f.name].map(
-              (value) => processTextFieldValue(value, fieldTextIndexLength)
-            );
-          } else {
-            data[f.name] = processTextFieldValue(
-              data[f.name],
-              fieldTextIndexLength
-            );
-          }
-        } else if (f.type === "rich-text") {
-          const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
-          if (isList) {
-            data[f.name] = data[f.name].map((value) => {
-              const acc = new StringBuilder(fieldTextIndexLength);
-              extractText(value, acc, ["text", "code_block", "html"]);
-              return acc.toString();
-            });
-          } else {
-            const acc = new StringBuilder(fieldTextIndexLength);
-            extractText(data[f.name], acc, ["text", "code_block", "html"]);
-            data[f.name] = acc.toString();
-          }
-        }
-      }
-    }
-    return data;
-  };
-  const memo = {};
-  const lookupStopwords = (keys, defaultStopWords = sw__namespace.eng) => {
-    let stopwords = defaultStopWords;
-    if (keys) {
-      if (memo[keys.join(",")]) {
-        return memo[keys.join(",")];
-      }
-      stopwords = [];
-      for (const key of keys) {
-        stopwords.push(...sw__namespace[key]);
-      }
-      memo[keys.join(",")] = stopwords;
-    }
-    return stopwords;
-  };
-  const queryToSearchIndexQuery = (query, stopwordLanguages) => {
-    let q;
-    const parts = query.split(" ");
-    const stopwords = lookupStopwords(stopwordLanguages);
-    if (parts.length === 1) {
-      q = { AND: [parts[0]] };
-    } else {
-      q = {
-        AND: parts.filter(
-          (part) => part.toLowerCase() !== "and" && stopwords.indexOf(part.toLowerCase()) === -1
-        )
-      };
     }
-    return q;
-  };
-  const optionsToSearchIndexOptions = (options) => {
-    const opt = {};
-    if (options == null ? void 0 : options.limit) {
-      opt["PAGE"] = {
-        SIZE: options.limit,
-        NUMBER: (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0
-      };
+  }
+  return data;
+};
+const stopwordCache = {};
+const lookupStopwords = (keys, defaultStopWords = sw.eng) => {
+  if (!keys) {
+    return defaultStopWords;
+  }
+  const cacheKey = keys.join(",");
+  if (stopwordCache[cacheKey]) {
+    return stopwordCache[cacheKey];
+  }
+  const stopwords = keys.flatMap((key) => sw[key] || []);
+  stopwordCache[cacheKey] = stopwords;
+  return stopwords;
+};
+const queryToSearchIndexQuery = (query, stopwordLanguages) => {
+  const parts = query.split(" ");
+  const stopwords = lookupStopwords(stopwordLanguages);
+  if (parts.length === 1) {
+    return { AND: [parts[0]] };
+  }
+  const filteredParts = parts.filter(
+    (part) => part.toLowerCase() !== "and" && !stopwords.includes(part.toLowerCase())
+  );
+  return { AND: filteredParts };
+};
+const optionsToSearchIndexOptions = (options) => {
+  if (!(options == null ? void 0 : options.limit))
+    return {};
+  return {
+    PAGE: {
+      SIZE: options.limit,
+      NUMBER: options.cursor ? parseInt(options.cursor) : 0
     }
-    return opt;
   };
-  const parseSearchIndexResponse = (data, options) => {
-    const results = data["RESULT"];
-    const total = data["RESULT_LENGTH"];
-    if ((options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
-      const prevCursor = options.cursor === "0" ? null : (parseInt(options.cursor) - 1).toString();
-      const nextCursor = total <= (parseInt(options.cursor) + 1) * options.limit ? null : (parseInt(options.cursor) + 1).toString();
-      return {
-        results,
-        total,
-        prevCursor,
-        nextCursor
-      };
-    } else if (!(options == null ? void 0 : options.cursor) && (options == null ? void 0 : options.limit)) {
-      const prevCursor = null;
-      const nextCursor = total <= options.limit ? null : "1";
-      return {
-        results,
-        total,
-        prevCursor,
-        nextCursor
-      };
-    } else {
-      return {
-        results,
-        total,
-        prevCursor: null,
-        nextCursor: null
-      };
-    }
+};
+const parseSearchIndexResponse = (data, options) => {
+  const resultArray = (data == null ? void 0 : data.RESULT) ?? (data == null ? void 0 : data.results);
+  if (!data || !Array.isArray(resultArray)) {
+    return {
+      results: [],
+      total: 0,
+      prevCursor: null,
+      nextCursor: null,
+      fuzzyMatches: void 0
+    };
+  }
+  const results = data.RESULT ?? data.results;
+  const total = data.RESULT_LENGTH ?? data.total ?? 0;
+  const fuzzyMatches = data.FUZZY_MATCHES ?? data.fuzzyMatches;
+  const nextCursor = data.NEXT_CURSOR ?? data.nextCursor;
+  const prevCursor = data.PREV_CURSOR ?? data.prevCursor;
+  if (nextCursor !== void 0 || prevCursor !== void 0) {
+    return {
+      results,
+      total,
+      prevCursor: prevCursor ?? null,
+      nextCursor: nextCursor ?? null,
+      fuzzyMatches
+    };
+  }
+  const currentPage = (options == null ? void 0 : options.cursor) ? parseInt(options.cursor) : 0;
+  const pageSize = options == null ? void 0 : options.limit;
+  const hasPreviousPage = currentPage > 0;
+  const hasNextPage = pageSize ? total > (currentPage + 1) * pageSize : false;
+  return {
+    results,
+    total,
+    prevCursor: hasPreviousPage ? (currentPage - 1).toString() : null,
+    nextCursor: hasNextPage ? (currentPage + 1).toString() : null,
+    fuzzyMatches
   };
-  exports2.optionsToSearchIndexOptions = optionsToSearchIndexOptions;
-  exports2.parseSearchIndexResponse = parseSearchIndexResponse;
-  exports2.processDocumentForIndexing = processDocumentForIndexing;
-  exports2.queryToSearchIndexQuery = queryToSearchIndexQuery;
-  Object.defineProperty(exports2, Symbol.toStringTag, { value: "Module" });
-});
+};
+export {
+  optionsToSearchIndexOptions,
+  parseSearchIndexResponse,
+  processDocumentForIndexing,
+  queryToSearchIndexQuery
+};

package/dist/index.d.ts

@@ -1,5 +1,10 @@
-import si from 'search-index';
+import createSearchIndex from 'search-index';
 export { SearchIndexer } from './indexer';
 export { LocalSearchIndexClient, TinaCMSSearchIndexClient } from './client';
-export type { SearchClient } from './types';
-export { si };
+export type { SearchClient, SearchOptions, SearchResult, SearchQueryResponse, IndexableDocument, SearchIndexResult, SearchIndex, } from './types';
+export type { FuzzySearchOptions, FuzzyMatch } from './fuzzy';
+export { levenshteinDistance, similarityScore, damerauLevenshteinDistance, findSimilarTerms, FuzzyCache, DEFAULT_FUZZY_OPTIONS, } from './fuzzy';
+export { FuzzySearchWrapper } from './fuzzy-search-wrapper';
+export { buildPageOptions, buildPaginationCursors } from './pagination';
+export type { PaginationOptions, PageOptions, PaginationCursors, } from './pagination';
+export { createSearchIndex };