@tinacms/search 0.0.0-00d7b85-20251222014449

package/dist/index.js

@@ -0,0 +1,683 @@
+// src/index.ts
+import si2 from "search-index";
+
+// src/indexer/index.ts
+import {
+  loadAndParseWithAliases,
+  sequential,
+  scanAllContent,
+  scanContentByPaths,
+  transformDocument,
+  transformDocumentIntoPayload
+} from "@tinacms/graphql";
+
+// src/indexer/utils.ts
+import * as sw from "stopword";
+var INDEXABLE_NODE_TYPES = ["text", "code_block", "html"];
+var StringBuilder = class {
+  buffer = [];
+  limit;
+  length = 0;
+  constructor(limit) {
+    this.limit = limit;
+  }
+  append(str) {
+    if (this.length + str.length > this.limit) return true;
+    this.buffer.push(str);
+    this.length += str.length;
+    return this.length > this.limit;
+  }
+  toString() {
+    return this.buffer.join(" ");
+  }
+};
+var tokenizeString = (str) => {
+  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+};
+var extractText = (data, builder, nodeTypes) => {
+  if (!data) return;
+  if (nodeTypes.includes(data.type ?? "") && (data.text || data.value)) {
+    const tokens = tokenizeString(data.text || data.value || "");
+    for (const token of tokens) {
+      if (builder.append(token)) return;
+    }
+  }
+  data.children?.forEach((child) => extractText(child, builder, nodeTypes));
+};
+var getRelativePath = (path, collection) => {
+  return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
+};
+var processTextField = (value, maxLength) => {
+  const tokens = tokenizeString(value);
+  const builder = new StringBuilder(maxLength);
+  for (const part of tokens) {
+    if (builder.append(part)) break;
+  }
+  return builder.toString();
+};
+var processRichTextField = (value, maxLength) => {
+  const builder = new StringBuilder(maxLength);
+  extractText(value, builder, INDEXABLE_NODE_TYPES);
+  return builder.toString();
+};
+var processObjectField = (data, path, collection, textIndexLength, field) => {
+  if (field.list) {
+    return data.map(
+      (obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, field)
+    );
+  }
+  return processDocumentForIndexing(
+    data,
+    path,
+    collection,
+    textIndexLength,
+    field
+  );
+};
+var processStringField = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processTextField(value, maxLength)
+    );
+  }
+  return processTextField(data, maxLength);
+};
+var processRichTextFieldData = (data, maxLength, isList) => {
+  if (isList) {
+    return data.map(
+      (value) => processRichTextField(value, maxLength)
+    );
+  }
+  return processRichTextField(data, maxLength);
+};
+var processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
+  if (!field) {
+    const relativePath = getRelativePath(path, collection);
+    data["_id"] = `${collection.name}:${relativePath}`;
+    data["_relativePath"] = relativePath;
+  }
+  const fields = field?.fields || collection.fields || [];
+  for (const f of fields) {
+    if (!f.searchable) {
+      delete data[f.name];
+      continue;
+    }
+    if (!data[f.name]) continue;
+    const fieldMaxLength = f.maxSearchIndexFieldLength || textIndexLength;
+    const isList = Boolean(f.list);
+    switch (f.type) {
+      case "object":
+        data[f.name] = processObjectField(
+          data[f.name],
+          path,
+          collection,
+          textIndexLength,
+          f
+        );
+        break;
+      case "string":
+        data[f.name] = processStringField(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
+        break;
+      case "rich-text":
+        data[f.name] = processRichTextFieldData(
+          data[f.name],
+          fieldMaxLength,
+          isList
+        );
+        break;
+    }
+  }
+  return data;
+};
+var stopwordCache = {};
+var lookupStopwords = (keys, defaultStopWords = sw.eng) => {
+  if (!keys) {
+    return defaultStopWords;
+  }
+  const cacheKey = keys.join(",");
+  if (stopwordCache[cacheKey]) {
+    return stopwordCache[cacheKey];
+  }
+  const stopwords = keys.flatMap((key) => sw[key] || []);
+  stopwordCache[cacheKey] = stopwords;
+  return stopwords;
+};
+
+// src/indexer/index.ts
+var SearchIndexer = class {
+  batchSize;
+  client;
+  bridge;
+  schema;
+  textIndexLength;
+  constructor(options) {
+    this.client = options.client;
+    this.bridge = options.bridge;
+    this.schema = options.schema;
+    this.batchSize = options.batchSize || 100;
+    this.textIndexLength = options.textIndexLength || 500;
+  }
+  createBatchProcessor() {
+    let batch = [];
+    return {
+      callback: async (item) => {
+        batch.push(item);
+        if (batch.length >= this.batchSize) {
+          await this.client.put(batch);
+          batch = [];
+        }
+      },
+      flush: async () => {
+        if (batch.length > 0) {
+          await this.client.put(batch);
+          batch = [];
+        }
+      }
+    };
+  }
+  makeIndexerCallback(itemCallback) {
+    return async (collection, contentPaths) => {
+      const templateInfo = this.schema.getTemplatesForCollectable(collection);
+      await sequential(contentPaths, async (path) => {
+        const data = await transformDocumentIntoPayload(
+          `${collection.path}/${path}`,
+          transformDocument(
+            path,
+            await loadAndParseWithAliases(
+              this.bridge,
+              path,
+              collection,
+              templateInfo
+            ),
+            this.schema
+          ),
+          this.schema
+        );
+        await itemCallback(
+          processDocumentForIndexing(
+            data["_values"],
+            path,
+            collection,
+            this.textIndexLength
+          )
+        );
+      });
+    };
+  }
+  async indexContentByPaths(documentPaths) {
+    const { callback, flush } = this.createBatchProcessor();
+    await this.client.onStartIndexing?.();
+    await scanContentByPaths(
+      this.schema,
+      documentPaths,
+      this.makeIndexerCallback(callback)
+    );
+    await flush();
+    await this.client.onFinishIndexing?.();
+  }
+  async indexAllContent() {
+    const { callback, flush } = this.createBatchProcessor();
+    await this.client.onStartIndexing?.();
+    const warnings = await scanAllContent(
+      this.schema,
+      this.bridge,
+      this.makeIndexerCallback(callback)
+    );
+    await flush();
+    await this.client.onFinishIndexing?.();
+    return { warnings };
+  }
+  async deleteIndexContent(documentPaths) {
+    await this.client.onStartIndexing?.();
+    await this.client.del(documentPaths);
+    await this.client.onFinishIndexing?.();
+  }
+};
+
+// src/client/index.ts
+import * as sqliteLevelModule from "sqlite-level";
+import si from "search-index";
+import { MemoryLevel } from "memory-level";
+
+// src/fuzzy/types.ts
+var DEFAULT_FUZZY_OPTIONS = {
+  maxDistance: 2,
+  minSimilarity: 0.6,
+  maxResults: 10,
+  useTranspositions: true,
+  caseSensitive: false,
+  useNgramFilter: true,
+  ngramSize: 2,
+  minNgramOverlap: 0.2
+};
+
+// src/fuzzy/cache.ts
+var FuzzyCache = class {
+  cache;
+  maxSize;
+  constructor(maxSize = 100) {
+    this.cache = /* @__PURE__ */ new Map();
+    this.maxSize = maxSize;
+  }
+  getCacheKey(query, options) {
+    return JSON.stringify({ query, options });
+  }
+  get(query, options) {
+    const key = this.getCacheKey(query, options);
+    const value = this.cache.get(key);
+    if (value) {
+      this.cache.delete(key);
+      this.cache.set(key, value);
+    }
+    return value;
+  }
+  set(query, options, results) {
+    const key = this.getCacheKey(query, options);
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+    this.cache.set(key, results);
+  }
+  clear() {
+    this.cache.clear();
+  }
+  get size() {
+    return this.cache.size;
+  }
+};
+
+// src/fuzzy/distance.ts
+var PREFIX_MATCH_MIN_SIMILARITY = 0.8;
+function levenshteinDistance(str1, str2) {
+  const len1 = str1.length;
+  const len2 = str2.length;
+  const dp = Array(len1 + 1).fill(null).map(() => Array(len2 + 1).fill(0));
+  for (let i = 0; i <= len1; i++) dp[i][0] = i;
+  for (let j = 0; j <= len2; j++) dp[0][j] = j;
+  for (let i = 1; i <= len1; i++) {
+    for (let j = 1; j <= len2; j++) {
+      if (str1[i - 1] === str2[j - 1]) {
+        dp[i][j] = dp[i - 1][j - 1];
+      } else {
+        dp[i][j] = Math.min(
+          dp[i - 1][j] + 1,
+          dp[i][j - 1] + 1,
+          dp[i - 1][j - 1] + 1
+        );
+      }
+    }
+  }
+  return dp[len1][len2];
+}
+function similarityScore(str1, str2, useTranspositions = false) {
+  const distance = useTranspositions ? damerauLevenshteinDistance(str1, str2) : levenshteinDistance(str1, str2);
+  const maxLength = Math.max(str1.length, str2.length);
+  if (maxLength === 0) return 1;
+  return 1 - distance / maxLength;
+}
+function damerauLevenshteinDistance(str1, str2) {
+  const len1 = str1.length;
+  const len2 = str2.length;
+  const maxDist = len1 + len2;
+  const charLastPosition = {};
+  const dp = Array(len1 + 2).fill(null).map(() => Array(len2 + 2).fill(0));
+  dp[0][0] = maxDist;
+  for (let i = 0; i <= len1; i++) {
+    dp[i + 1][0] = maxDist;
+    dp[i + 1][1] = i;
+  }
+  for (let j = 0; j <= len2; j++) {
+    dp[0][j + 1] = maxDist;
+    dp[1][j + 1] = j;
+  }
+  for (let i = 1; i <= len1; i++) {
+    let lastMatchingCol = 0;
+    for (let j = 1; j <= len2; j++) {
+      const lastRowWithMatch = charLastPosition[str2[j - 1]] || 0;
+      const lastColWithMatch = lastMatchingCol;
+      let cost = 1;
+      if (str1[i - 1] === str2[j - 1]) {
+        cost = 0;
+        lastMatchingCol = j;
+      }
+      dp[i + 1][j + 1] = Math.min(
+        dp[i][j] + cost,
+        dp[i + 1][j] + 1,
+        dp[i][j + 1] + 1,
+        dp[lastRowWithMatch][lastColWithMatch] + (i - lastRowWithMatch - 1) + 1 + (j - lastColWithMatch - 1)
+      );
+    }
+    charLastPosition[str1[i - 1]] = i;
+  }
+  return dp[len1 + 1][len2 + 1];
+}
+function getNgrams(str, n = 2) {
+  const ngrams = /* @__PURE__ */ new Set();
+  if (str.length < n) {
+    ngrams.add(str);
+    return ngrams;
+  }
+  for (let i = 0; i <= str.length - n; i++) {
+    ngrams.add(str.substring(i, i + n));
+  }
+  return ngrams;
+}
+function ngramOverlap(ngrams1, ngrams2) {
+  if (ngrams1.size === 0 || ngrams2.size === 0) return 0;
+  let overlap = 0;
+  for (const ngram of ngrams1) {
+    if (ngrams2.has(ngram)) overlap++;
+  }
+  const minSize = Math.min(ngrams1.size, ngrams2.size);
+  return overlap / minSize;
+}
+function findSimilarTerms(query, dictionary, options = {}) {
+  const opts = { ...DEFAULT_FUZZY_OPTIONS, ...options };
+  const normalizedQuery = opts.caseSensitive ? query : query.toLowerCase();
+  if (normalizedQuery.length === 0) return [];
+  const matches = [];
+  const distanceFunc = opts.useTranspositions ? damerauLevenshteinDistance : levenshteinDistance;
+  const queryNgrams = opts.useNgramFilter ? getNgrams(normalizedQuery, opts.ngramSize) : null;
+  for (const term of dictionary) {
+    if (typeof term !== "string" || term.length === 0) continue;
+    const normalizedTerm = opts.caseSensitive ? term : term.toLowerCase();
+    if (queryNgrams) {
+      const termNgrams = getNgrams(normalizedTerm, opts.ngramSize);
+      const overlap = ngramOverlap(queryNgrams, termNgrams);
+      if (overlap < opts.minNgramOverlap) continue;
+    }
+    if (normalizedTerm.startsWith(normalizedQuery)) {
+      const prefixSimilarity = normalizedQuery.length / normalizedTerm.length;
+      matches.push({
+        term,
+        distance: normalizedTerm.length - normalizedQuery.length,
+        similarity: Math.max(prefixSimilarity, PREFIX_MATCH_MIN_SIMILARITY)
+      });
+      continue;
+    }
+    const distance = distanceFunc(normalizedQuery, normalizedTerm);
+    if (distance > opts.maxDistance) continue;
+    const similarity = similarityScore(
+      normalizedQuery,
+      normalizedTerm,
+      opts.useTranspositions
+    );
+    if (similarity >= opts.minSimilarity) {
+      matches.push({ term, distance, similarity });
+    }
+  }
+  matches.sort((a, b) => {
+    if (Math.abs(a.similarity - b.similarity) < 1e-3) {
+      return a.distance - b.distance;
+    }
+    return b.similarity - a.similarity;
+  });
+  return matches.slice(0, opts.maxResults);
+}
+
+// src/pagination.ts
+function buildPageOptions(options) {
+  if (!options.limit) return {};
+  return {
+    PAGE: {
+      NUMBER: options.cursor ? parseInt(options.cursor, 10) : 0,
+      SIZE: options.limit
+    }
+  };
+}
+function buildPaginationCursors(total, options) {
+  const currentPage = options.cursor ? parseInt(options.cursor, 10) : 0;
+  const pageSize = options.limit;
+  const hasPreviousPage = currentPage > 0;
+  const hasNextPage = pageSize ? total > (currentPage + 1) * pageSize : false;
+  return {
+    prevCursor: hasPreviousPage ? (currentPage - 1).toString() : null,
+    nextCursor: hasNextPage ? (currentPage + 1).toString() : null
+  };
+}
+
+// src/fuzzy-search-wrapper.ts
+var FuzzySearchWrapper = class {
+  cache;
+  searchIndex;
+  constructor(searchIndex, cacheSize = 100) {
+    this.searchIndex = searchIndex;
+    this.cache = new FuzzyCache(cacheSize);
+  }
+  async getDictionary(field) {
+    const token = field ? { FIELD: field } : void 0;
+    const dictionary = await this.searchIndex.DICTIONARY(token);
+    return dictionary.filter((entry) => typeof entry === "string");
+  }
+  async findSimilar(query, field, options = {}) {
+    const cacheKey = `${query}:${field || "all"}`;
+    const cached = this.cache.get(cacheKey, options);
+    if (cached) return cached;
+    const dictionary = await this.getDictionary(field);
+    const matches = findSimilarTerms(query, dictionary, options);
+    this.cache.set(cacheKey, options, matches);
+    return matches;
+  }
+  async expandQuery(query, options = {}) {
+    const opts = { ...DEFAULT_FUZZY_OPTIONS, ...options };
+    const terms = query.split(" ").map((t) => t.trim()).filter((t) => t.length > 0);
+    const expanded = [];
+    const matches = {};
+    for (const term of terms) {
+      const similarTerms = await this.findSimilar(term, void 0, opts);
+      expanded.push(term);
+      const similarValues = similarTerms.filter((m) => m.term.toLowerCase() !== term.toLowerCase()).map((m) => m.term);
+      expanded.push(...similarValues);
+      if (similarTerms.length > 0) {
+        matches[term] = similarTerms;
+      }
+    }
+    return {
+      original: terms,
+      expanded: Array.from(new Set(expanded)),
+      matches
+    };
+  }
+  async query(query, options = {}) {
+    const pageOptions = buildPageOptions(options);
+    const expansion = await this.expandQuery(query, options.fuzzyOptions);
+    if (expansion.expanded.length === expansion.original.length) {
+      const results2 = await this.searchIndex.QUERY(
+        { AND: expansion.original },
+        pageOptions
+      );
+      const pagination2 = buildPaginationCursors(
+        results2.RESULT_LENGTH || 0,
+        options
+      );
+      return {
+        results: results2.RESULT || [],
+        total: results2.RESULT_LENGTH || 0,
+        ...pagination2,
+        fuzzyMatches: expansion.matches
+      };
+    }
+    const queryGroups = expansion.original.map((originalTerm) => {
+      const similarTerms = expansion.matches[originalTerm]?.map((m) => m.term) || [];
+      return [originalTerm, ...similarTerms];
+    });
+    const searchQuery = queryGroups.length === 1 ? { OR: queryGroups[0] } : {
+      AND: queryGroups.map(
+        (group) => group.length === 1 ? group[0] : { OR: group }
+      )
+    };
+    const results = await this.searchIndex.QUERY(searchQuery, pageOptions);
+    const pagination = buildPaginationCursors(
+      results.RESULT_LENGTH || 0,
+      options
+    );
+    return {
+      results: results.RESULT || [],
+      total: results.RESULT_LENGTH || 0,
+      ...pagination,
+      fuzzyMatches: expansion.matches
+    };
+  }
+  clearCache() {
+    this.cache.clear();
+  }
+  getCacheSize() {
+    return this.cache.size;
+  }
+};
+
+// src/client/index.ts
+import * as zlib from "node:zlib";
+var SqliteLevel2 = sqliteLevelModule.default?.SqliteLevel ?? sqliteLevelModule.SqliteLevel;
+var DEFAULT_TOKEN_SPLIT_REGEX = /[\p{L}\d_]+/gu;
+var LocalSearchIndexClient = class {
+  searchIndex;
+  memoryLevel;
+  stopwords;
+  tokenSplitRegex;
+  fuzzySearchWrapper;
+  constructor(options) {
+    this.memoryLevel = new MemoryLevel();
+    this.stopwords = lookupStopwords(options.stopwordLanguages);
+    this.tokenSplitRegex = options.tokenSplitRegex ? new RegExp(options.tokenSplitRegex, "gu") : DEFAULT_TOKEN_SPLIT_REGEX;
+  }
+  async onStartIndexing() {
+    const options = {
+      db: this.memoryLevel,
+      stopwords: this.stopwords,
+      tokenSplitRegex: this.tokenSplitRegex
+    };
+    this.searchIndex = await si(
+      options
+    );
+    this.fuzzySearchWrapper = new FuzzySearchWrapper(this.searchIndex);
+  }
+  async put(docs) {
+    if (!this.searchIndex) {
+      throw new Error("onStartIndexing must be called first");
+    }
+    await this.searchIndex.PUT(docs);
+  }
+  async del(ids) {
+    if (!this.searchIndex) {
+      throw new Error("onStartIndexing must be called first");
+    }
+    await this.searchIndex.DELETE(ids);
+  }
+  async query(query, options) {
+    if (!this.searchIndex) {
+      throw new Error("onStartIndexing must be called first");
+    }
+    if (options?.fuzzy && this.fuzzySearchWrapper) {
+      return this.fuzzySearchWrapper.query(query, {
+        limit: options.limit,
+        cursor: options.cursor,
+        fuzzyOptions: options.fuzzyOptions
+      });
+    }
+    const searchIndexOptions = buildPageOptions({
+      limit: options?.limit,
+      cursor: options?.cursor
+    });
+    const terms = query.split(" ").filter((t) => t.trim().length > 0);
+    const queryObj = terms.length > 1 ? { AND: terms } : { AND: [terms[0] || ""] };
+    const searchResults = await this.searchIndex.QUERY(
+      queryObj,
+      searchIndexOptions
+    );
+    const total = searchResults.RESULT_LENGTH || 0;
+    const pagination = buildPaginationCursors(total, {
+      limit: options?.limit,
+      cursor: options?.cursor
+    });
+    return {
+      results: searchResults.RESULT || [],
+      total,
+      ...pagination
+    };
+  }
+  async export(filename) {
+    const sqliteLevel = new SqliteLevel2({ filename });
+    const iterator = this.memoryLevel.iterator();
+    for await (const [key, value] of iterator) {
+      await sqliteLevel.put(key, value);
+    }
+    await sqliteLevel.close();
+  }
+};
+var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
+  apiUrl;
+  branch;
+  indexerToken;
+  constructor(options) {
+    super(options);
+    this.apiUrl = options.apiUrl;
+    this.branch = options.branch;
+    this.indexerToken = options.indexerToken;
+  }
+  async getUploadUrl() {
+    const headers = new Headers();
+    headers.append("x-api-key", this.indexerToken || "");
+    headers.append("Content-Type", "application/json");
+    const response = await fetch(`${this.apiUrl}/upload/${this.branch}`, {
+      method: "GET",
+      headers
+    });
+    if (response.status !== 200) {
+      const errorBody = await response.json().catch(() => ({}));
+      throw new Error(
+        `Failed to get upload url. Status: ${response.status}${errorBody?.message ? ` - ${errorBody.message}` : ""}`
+      );
+    }
+    const { signedUrl } = await response.json();
+    return signedUrl;
+  }
+  async serializeIndex() {
+    const sqliteLevel = new SqliteLevel2({ filename: ":memory:" });
+    const iterator = this.memoryLevel.iterator();
+    for await (const [key, value] of iterator) {
+      await sqliteLevel.put(key, value);
+    }
+    const buffer = sqliteLevel.db.serialize();
+    await sqliteLevel.close();
+    return zlib.gzipSync(buffer);
+  }
+  async uploadIndex(signedUrl, data) {
+    const response = await fetch(signedUrl, {
+      method: "PUT",
+      body: data
+    });
+    if (response.status !== 200) {
+      const errorText = await response.text();
+      throw new Error(
+        `Failed to upload search index. Status: ${response.status}
+${errorText}`
+      );
+    }
+  }
+  async onFinishIndexing() {
+    const signedUrl = await this.getUploadUrl();
+    const indexData = await this.serializeIndex();
+    await this.uploadIndex(signedUrl, indexData);
+  }
+};
+export {
+  DEFAULT_FUZZY_OPTIONS,
+  FuzzyCache,
+  FuzzySearchWrapper,
+  LocalSearchIndexClient,
+  SearchIndexer,
+  TinaCMSSearchIndexClient,
+  buildPageOptions,
+  buildPaginationCursors,
+  damerauLevenshteinDistance,
+  findSimilarTerms,
+  levenshteinDistance,
+  si2 as si,
+  similarityScore
+};

package/dist/indexer/index.d.ts

@@ -0,0 +1,26 @@
+import { TinaSchema } from '@tinacms/schema-tools';
+import { Bridge } from '@tinacms/graphql';
+import { SearchClient } from '../types';
+type SearchIndexOptions = {
+    batchSize?: number;
+    bridge: Bridge;
+    client: SearchClient;
+    schema: TinaSchema;
+    textIndexLength?: number;
+};
+export declare class SearchIndexer {
+    private readonly batchSize;
+    private readonly client;
+    private readonly bridge;
+    private readonly schema;
+    private readonly textIndexLength;
+    constructor(options: SearchIndexOptions);
+    private createBatchProcessor;
+    private makeIndexerCallback;
+    indexContentByPaths(documentPaths: string[]): Promise<void>;
+    indexAllContent(): Promise<{
+        warnings: string[];
+    }>;
+    deleteIndexContent(documentPaths: string[]): Promise<void>;
+}
+export {};

package/dist/indexer/utils.d.ts

@@ -0,0 +1,3 @@
+import { Collection, ObjectField } from '@tinacms/schema-tools';
+export declare const processDocumentForIndexing: (data: Record<string, unknown>, path: string, collection: Collection, textIndexLength: number, field?: ObjectField) => Record<string, unknown>;
+export declare const lookupStopwords: (keys?: string[], defaultStopWords?: string[]) => string[];