@tinacms/search 1.0.2 → 1.0.4

package/dist/client/index.d.ts

@@ -1,7 +1,21 @@
 import type { SearchClient } from '../types';
+import { MemoryLevel } from 'memory-level';
+declare type TinaSearchIndexerClientOptions = {
+    stopwordLanguages?: string[];
+    tokenSplitRegex?: string;
+};
+declare type TinaCloudSearchIndexerClientOptions = {
+    apiUrl: string;
+    branch: string;
+    indexerToken: string;
+} & TinaSearchIndexerClientOptions;
 export declare class LocalSearchIndexClient implements SearchClient {
-    protected searchIndex: any;
-    constructor(searchIndex: any);
+    searchIndex: any;
+    protected readonly memoryLevel: MemoryLevel;
+    private readonly stopwords;
+    private readonly tokenSplitRegex;
+    constructor(options: TinaSearchIndexerClientOptions);
+    onStartIndexing(): Promise<void>;
     put(docs: any[]): Promise<any>;
     del(ids: string[]): Promise<any>;
     query(query: string, options: {
@@ -13,34 +27,13 @@ export declare class LocalSearchIndexClient implements SearchClient {
         nextCursor: string | null;
         prevCursor: string | null;
     }>;
+    export(filename: string): Promise<void>;
 }
-declare type TinaCloudSearchIndexerClientOptions = {
-    apiUrl: string;
-    branch: string;
-    indexerToken: string;
-    stopwordLanguages?: string[];
-};
-export declare class TinaCMSSearchIndexClient implements SearchClient {
-    private memoryLevel;
-    private searchIndex;
+export declare class TinaCMSSearchIndexClient extends LocalSearchIndexClient {
     private readonly apiUrl;
     private readonly branch;
     private readonly indexerToken;
-    private readonly stopwordLanguages;
     constructor(options: TinaCloudSearchIndexerClientOptions);
-    put(docs: any[]): Promise<void>;
-    del(ids: string[]): Promise<void>;
-    onStartIndexing(): Promise<void>;
     onFinishIndexing(): Promise<void>;
-    getSearchIndex(): any;
-    query(query: string, options: {
-        cursor?: string;
-        limit?: number;
-    } | undefined): Promise<{
-        results: any[];
-        total: number;
-        nextCursor: string | null;
-        prevCursor: string | null;
-    }>;
 }
 export {};

package/dist/index-client.es.js

@@ -31,7 +31,10 @@ const extractText = (data, acc, indexableNodeTypes) => {
         }
       }
     }
-    (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(_a, (child) => extractText(child, acc, indexableNodeTypes));
+    (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
+      _a,
+      (child) => extractText(child, acc, indexableNodeTypes)
+    );
   }
 };
 const relativePath = (path, collection) => {
@@ -65,16 +68,35 @@ const processDocumentForIndexing = (data, path, collection, textIndexLength, fie
     if (data[f.name]) {
       if (f.type === "object") {
         if (isList) {
-          data[f.name] = data[f.name].map((obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, f));
+          data[f.name] = data[f.name].map(
+            (obj) => processDocumentForIndexing(
+              obj,
+              path,
+              collection,
+              textIndexLength,
+              f
+            )
+          );
         } else {
-          data[f.name] = processDocumentForIndexing(data[f.name], path, collection, textIndexLength, f);
+          data[f.name] = processDocumentForIndexing(
+            data[f.name],
+            path,
+            collection,
+            textIndexLength,
+            f
+          );
         }
       } else if (f.type === "string") {
         const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
         if (isList) {
-          data[f.name] = data[f.name].map((value) => processTextFieldValue(value, fieldTextIndexLength));
+          data[f.name] = data[f.name].map(
+            (value) => processTextFieldValue(value, fieldTextIndexLength)
+          );
         } else {
-          data[f.name] = processTextFieldValue(data[f.name], fieldTextIndexLength);
+          data[f.name] = processTextFieldValue(
+            data[f.name],
+            fieldTextIndexLength
+          );
         }
       } else if (f.type === "rich-text") {
         const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
@@ -144,4 +166,9 @@ const parseSearchIndexResponse = (data, options) => {
     };
   }
 };
-export { optionsToSearchIndexOptions, parseSearchIndexResponse, processDocumentForIndexing, queryToSearchIndexQuery };
+export {
+  optionsToSearchIndexOptions,
+  parseSearchIndexResponse,
+  processDocumentForIndexing,
+  queryToSearchIndexQuery
+};

package/dist/index-client.js

@@ -35,7 +35,10 @@
           }
         }
       }
-      (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(_a, (child) => extractText(child, acc, indexableNodeTypes));
+      (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
+        _a,
+        (child) => extractText(child, acc, indexableNodeTypes)
+      );
     }
   };
   const relativePath = (path, collection) => {
@@ -69,16 +72,35 @@
       if (data[f.name]) {
         if (f.type === "object") {
           if (isList) {
-            data[f.name] = data[f.name].map((obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, f));
+            data[f.name] = data[f.name].map(
+              (obj) => processDocumentForIndexing(
+                obj,
+                path,
+                collection,
+                textIndexLength,
+                f
+              )
+            );
           } else {
-            data[f.name] = processDocumentForIndexing(data[f.name], path, collection, textIndexLength, f);
+            data[f.name] = processDocumentForIndexing(
+              data[f.name],
+              path,
+              collection,
+              textIndexLength,
+              f
+            );
           }
         } else if (f.type === "string") {
           const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
           if (isList) {
-            data[f.name] = data[f.name].map((value) => processTextFieldValue(value, fieldTextIndexLength));
+            data[f.name] = data[f.name].map(
+              (value) => processTextFieldValue(value, fieldTextIndexLength)
+            );
           } else {
-            data[f.name] = processTextFieldValue(data[f.name], fieldTextIndexLength);
+            data[f.name] = processTextFieldValue(
+              data[f.name],
+              fieldTextIndexLength
+            );
           }
         } else if (f.type === "rich-text") {
           const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
@@ -152,5 +174,5 @@
   exports2.parseSearchIndexResponse = parseSearchIndexResponse;
   exports2.processDocumentForIndexing = processDocumentForIndexing;
   exports2.queryToSearchIndexQuery = queryToSearchIndexQuery;
-  Object.defineProperties(exports2, { __esModule: { value: true }, [Symbol.toStringTag]: { value: "Module" } });
+  Object.defineProperty(exports2, Symbol.toStringTag, { value: "Module" });
 });

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-import si from '@tinacms/search-index';
+import si from 'search-index';
 export { SearchIndexer } from './indexer';
 export { LocalSearchIndexClient, TinaCMSSearchIndexClient } from './client';
 export type { SearchClient } from './types';

package/dist/index.js

@@ -34,7 +34,7 @@ __export(src_exports, {
 });
 module.exports = __toCommonJS(src_exports);
 var sw = __toESM(require("stopword"));
-var import_search_index2 = __toESM(require("@tinacms/search-index"));
+var import_search_index2 = __toESM(require("search-index"));
 
 // src/indexer/index.ts
 var import_graphql = require("@tinacms/graphql");
@@ -253,17 +253,33 @@ var SearchIndexer = class {
 // src/client/index.ts
 var import_sqlite_level = require("sqlite-level");
 var zlib = __toESM(require("zlib"));
-var import_search_index = __toESM(require("@tinacms/search-index"));
+var import_search_index = __toESM(require("search-index"));
 var import_memory_level = require("memory-level");
 var import_node_fetch = __toESM(require("node-fetch"));
+var DEFAULT_TOKEN_SPLIT_REGEX = /[\p{L}\d_]+/gu;
 var LocalSearchIndexClient = class {
-  constructor(searchIndex) {
-    this.searchIndex = searchIndex;
+  constructor(options) {
+    this.memoryLevel = new import_memory_level.MemoryLevel();
+    this.stopwords = lookupStopwords(options.stopwordLanguages);
+    this.tokenSplitRegex = options.tokenSplitRegex ? new RegExp(options.tokenSplitRegex, "gu") : DEFAULT_TOKEN_SPLIT_REGEX;
+  }
+  async onStartIndexing() {
+    this.searchIndex = await (0, import_search_index.default)({
+      db: this.memoryLevel,
+      stopwords: this.stopwords,
+      tokenSplitRegex: this.tokenSplitRegex
+    });
   }
   async put(docs) {
+    if (!this.searchIndex) {
+      throw new Error("onStartIndexing must be called first");
+    }
     return this.searchIndex.PUT(docs);
   }
   async del(ids) {
+    if (!this.searchIndex) {
+      throw new Error("onStartIndexing must be called first");
+    }
     return this.searchIndex.DELETE(ids);
   }
   query(query, options) {
@@ -274,30 +290,21 @@ var LocalSearchIndexClient = class {
       total: 0
     });
   }
+  async export(filename) {
+    const sqliteLevel = new import_sqlite_level.SqliteLevel({ filename });
+    const iterator = this.memoryLevel.iterator();
+    for await (const [key, value] of iterator) {
+      await sqliteLevel.put(key, value);
+    }
+    await sqliteLevel.close();
+  }
 };
-var TinaCMSSearchIndexClient = class {
+var TinaCMSSearchIndexClient = class extends LocalSearchIndexClient {
   constructor(options) {
+    super(options);
     this.apiUrl = options.apiUrl;
     this.branch = options.branch;
     this.indexerToken = options.indexerToken;
-    this.stopwordLanguages = options.stopwordLanguages;
-  }
-  async put(docs) {
-    if (this.searchIndex) {
-      await this.searchIndex.PUT(docs);
-    }
-  }
-  async del(ids) {
-    if (this.searchIndex) {
-      await this.searchIndex.DELETE(ids);
-    }
-  }
-  async onStartIndexing() {
-    this.memoryLevel = new import_memory_level.MemoryLevel();
-    this.searchIndex = await (0, import_search_index.default)({
-      db: this.memoryLevel,
-      stopwords: lookupStopwords(this.stopwordLanguages)
-    });
   }
   async onFinishIndexing() {
     const headers = new import_node_fetch.Headers();
@@ -337,17 +344,6 @@ ${await res.text()}`
       );
     }
   }
-  getSearchIndex() {
-    return this.searchIndex;
-  }
-  query(query, options) {
-    return Promise.resolve({
-      nextCursor: void 0,
-      prevCursor: void 0,
-      results: [],
-      total: 0
-    });
-  }
 };
 
 // src/index.ts

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tinacms/search",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "main": "dist/index.js",
   "module": "dist/index-client.es.js",
   "typings": "dist/index.d.ts",
@@ -29,13 +29,13 @@
     ]
   },
   "dependencies": {
-    "@tinacms/graphql": "1.4.17",
-    "@tinacms/schema-tools": "1.4.6",
+    "@tinacms/graphql": "1.4.19",
+    "@tinacms/schema-tools": "1.4.7",
     "abstract-level": "^1.0.3",
     "memory-level": "^1.0.0",
     "module-error": "^1.0.2",
     "node-fetch": "2",
-    "@tinacms/search-index": "^3.5.2",
+    "search-index": "4.0.0",
     "stopword": "^2.0.8",
     "sqlite-level": "^1.0.1"
   },
@@ -51,10 +51,10 @@
     "@types/jest": "^26.0.4",
     "@types/micromatch": "^4.0.2",
     "@types/search-index": "^3.2.0",
-    "jest": "27.0.6",
-    "jest-diff": "27.0.6",
+    "jest": "^29.5.0",
+    "jest-diff": "^29.5.0",
     "jest-file-snapshot": "^0.5.0",
-    "jest-matcher-utils": "27.0.6",
+    "jest-matcher-utils": "^29.5.0",
     "typescript": "4.3.5"
   },
   "scripts": {