@tinacms/search 1.0.0 → 1.0.2

package/dist/index-client.d.ts

@@ -1,4 +1,6 @@
 export type { SearchClient } from './types';
+export { processDocumentForIndexing } from './indexer/utils';
+export declare const queryToSearchIndexQuery: (query: string) => any;
 export declare const optionsToSearchIndexOptions: (options?: {
     limit?: number;
     cursor?: string;

package/dist/index-client.es.js

@@ -1,3 +1,109 @@
+class StringBuilder {
+  constructor(limit) {
+    this.length = 0;
+    this.buffer = [];
+    this.limit = limit;
+  }
+  append(str) {
+    if (this.length + str.length > this.limit) {
+      return true;
+    } else {
+      this.buffer.push(str);
+      this.length += str.length;
+      if (this.length > this.limit) {
+        return true;
+      }
+      return false;
+    }
+  }
+  toString() {
+    return this.buffer.join(" ");
+  }
+}
+const extractText = (data, acc, indexableNodeTypes) => {
+  var _a, _b;
+  if (data) {
+    if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
+      const tokens = tokenizeString(data.text || data.value);
+      for (const token of tokens) {
+        if (acc.append(token)) {
+          return;
+        }
+      }
+    }
+    (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(_a, (child) => extractText(child, acc, indexableNodeTypes));
+  }
+};
+const relativePath = (path, collection) => {
+  return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
+};
+const tokenizeString = (str) => {
+  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+};
+const processTextFieldValue = (value, maxLen) => {
+  const tokens = tokenizeString(value);
+  const builder = new StringBuilder(maxLen);
+  for (const part of tokens) {
+    if (builder.append(part)) {
+      break;
+    }
+  }
+  return builder.toString();
+};
+const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
+  if (!field) {
+    const relPath = relativePath(path, collection);
+    data["_id"] = `${collection.name}:${relPath}`;
+    data["_relativePath"] = relPath;
+  }
+  for (const f of collection.fields || (field == null ? void 0 : field.fields) || []) {
+    if (!f.searchable) {
+      delete data[f.name];
+      continue;
+    }
+    const isList = f.list;
+    if (data[f.name]) {
+      if (f.type === "object") {
+        if (isList) {
+          data[f.name] = data[f.name].map((obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, f));
+        } else {
+          data[f.name] = processDocumentForIndexing(data[f.name], path, collection, textIndexLength, f);
+        }
+      } else if (f.type === "string") {
+        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+        if (isList) {
+          data[f.name] = data[f.name].map((value) => processTextFieldValue(value, fieldTextIndexLength));
+        } else {
+          data[f.name] = processTextFieldValue(data[f.name], fieldTextIndexLength);
+        }
+      } else if (f.type === "rich-text") {
+        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+        if (isList) {
+          data[f.name] = data[f.name].map((value) => {
+            const acc = new StringBuilder(fieldTextIndexLength);
+            extractText(value, acc, ["text", "code_block", "html"]);
+            return acc.toString();
+          });
+        } else {
+          const acc = new StringBuilder(fieldTextIndexLength);
+          extractText(data[f.name], acc, ["text", "code_block", "html"]);
+          data[f.name] = acc.toString();
+        }
+      }
+    }
+  }
+  return data;
+};
+const queryToSearchIndexQuery = (query) => {
+  let q;
+  const parts = query.split(" ");
+  if (parts.length === 1) {
+    q = { AND: [parts[0]] };
+  } else {
+    q = { AND: parts.filter((part) => part.toLowerCase() !== "and") };
+  }
+  return q;
+};
 const optionsToSearchIndexOptions = (options) => {
   const opt = {};
   if (options == null ? void 0 : options.limit) {
@@ -38,4 +144,4 @@ const parseSearchIndexResponse = (data, options) => {
     };
   }
 };
-export { optionsToSearchIndexOptions, parseSearchIndexResponse };
+export { optionsToSearchIndexOptions, parseSearchIndexResponse, processDocumentForIndexing, queryToSearchIndexQuery };

package/dist/index-client.js

@@ -2,6 +2,112 @@
   typeof exports === "object" && typeof module !== "undefined" ? factory(exports) : typeof define === "function" && define.amd ? define(["exports"], factory) : (global = typeof globalThis !== "undefined" ? globalThis : global || self, factory(global["@tinacms/search"] = {}));
 })(this, function(exports2) {
   "use strict";
+  class StringBuilder {
+    constructor(limit) {
+      this.length = 0;
+      this.buffer = [];
+      this.limit = limit;
+    }
+    append(str) {
+      if (this.length + str.length > this.limit) {
+        return true;
+      } else {
+        this.buffer.push(str);
+        this.length += str.length;
+        if (this.length > this.limit) {
+          return true;
+        }
+        return false;
+      }
+    }
+    toString() {
+      return this.buffer.join(" ");
+    }
+  }
+  const extractText = (data, acc, indexableNodeTypes) => {
+    var _a, _b;
+    if (data) {
+      if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
+        const tokens = tokenizeString(data.text || data.value);
+        for (const token of tokens) {
+          if (acc.append(token)) {
+            return;
+          }
+        }
+      }
+      (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(_a, (child) => extractText(child, acc, indexableNodeTypes));
+    }
+  };
+  const relativePath = (path, collection) => {
+    return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
+  };
+  const tokenizeString = (str) => {
+    return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+  };
+  const processTextFieldValue = (value, maxLen) => {
+    const tokens = tokenizeString(value);
+    const builder = new StringBuilder(maxLen);
+    for (const part of tokens) {
+      if (builder.append(part)) {
+        break;
+      }
+    }
+    return builder.toString();
+  };
+  const processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
+    if (!field) {
+      const relPath = relativePath(path, collection);
+      data["_id"] = `${collection.name}:${relPath}`;
+      data["_relativePath"] = relPath;
+    }
+    for (const f of collection.fields || (field == null ? void 0 : field.fields) || []) {
+      if (!f.searchable) {
+        delete data[f.name];
+        continue;
+      }
+      const isList = f.list;
+      if (data[f.name]) {
+        if (f.type === "object") {
+          if (isList) {
+            data[f.name] = data[f.name].map((obj) => processDocumentForIndexing(obj, path, collection, textIndexLength, f));
+          } else {
+            data[f.name] = processDocumentForIndexing(data[f.name], path, collection, textIndexLength, f);
+          }
+        } else if (f.type === "string") {
+          const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+          if (isList) {
+            data[f.name] = data[f.name].map((value) => processTextFieldValue(value, fieldTextIndexLength));
+          } else {
+            data[f.name] = processTextFieldValue(data[f.name], fieldTextIndexLength);
+          }
+        } else if (f.type === "rich-text") {
+          const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+          if (isList) {
+            data[f.name] = data[f.name].map((value) => {
+              const acc = new StringBuilder(fieldTextIndexLength);
+              extractText(value, acc, ["text", "code_block", "html"]);
+              return acc.toString();
+            });
+          } else {
+            const acc = new StringBuilder(fieldTextIndexLength);
+            extractText(data[f.name], acc, ["text", "code_block", "html"]);
+            data[f.name] = acc.toString();
+          }
+        }
+      }
+    }
+    return data;
+  };
+  const queryToSearchIndexQuery = (query) => {
+    let q;
+    const parts = query.split(" ");
+    if (parts.length === 1) {
+      q = { AND: [parts[0]] };
+    } else {
+      q = { AND: parts.filter((part) => part.toLowerCase() !== "and") };
+    }
+    return q;
+  };
   const optionsToSearchIndexOptions = (options) => {
     const opt = {};
     if (options == null ? void 0 : options.limit) {
@@ -44,5 +150,7 @@
   };
   exports2.optionsToSearchIndexOptions = optionsToSearchIndexOptions;
   exports2.parseSearchIndexResponse = parseSearchIndexResponse;
+  exports2.processDocumentForIndexing = processDocumentForIndexing;
+  exports2.queryToSearchIndexQuery = queryToSearchIndexQuery;
   Object.defineProperties(exports2, { __esModule: { value: true }, [Symbol.toStringTag]: { value: "Module" } });
 });

package/dist/index.d.ts

@@ -1,5 +1,7 @@
+import si from '@tinacms/search-index';
 export { SearchIndexer } from './indexer';
 export { LocalSearchIndexClient, TinaCMSSearchIndexClient } from './client';
 export type { SearchClient } from './types';
 export { default as sw } from 'stopword';
+export { si };
 export declare const lookupStopwords: (keys?: string[], defaultStopWords?: string[]) => string[];

package/dist/index.js

@@ -29,13 +29,137 @@ __export(src_exports, {
   SearchIndexer: () => SearchIndexer,
   TinaCMSSearchIndexClient: () => TinaCMSSearchIndexClient,
   lookupStopwords: () => lookupStopwords,
+  si: () => import_search_index2.default,
   sw: () => import_stopword.default
 });
 module.exports = __toCommonJS(src_exports);
 var sw = __toESM(require("stopword"));
+var import_search_index2 = __toESM(require("@tinacms/search-index"));
 
 // src/indexer/index.ts
 var import_graphql = require("@tinacms/graphql");
+
+// src/indexer/utils.ts
+var StringBuilder = class {
+  constructor(limit) {
+    this.length = 0;
+    this.buffer = [];
+    this.limit = limit;
+  }
+  append(str) {
+    if (this.length + str.length > this.limit) {
+      return true;
+    } else {
+      this.buffer.push(str);
+      this.length += str.length;
+      if (this.length > this.limit) {
+        return true;
+      }
+      return false;
+    }
+  }
+  toString() {
+    return this.buffer.join(" ");
+  }
+};
+var extractText = (data, acc, indexableNodeTypes) => {
+  var _a, _b;
+  if (data) {
+    if (indexableNodeTypes.indexOf(data.type) !== -1 && (data.text || data.value)) {
+      const tokens = tokenizeString(data.text || data.value);
+      for (const token of tokens) {
+        if (acc.append(token)) {
+          return;
+        }
+      }
+    }
+    (_b = (_a = data.children) == null ? void 0 : _a.forEach) == null ? void 0 : _b.call(
+      _a,
+      (child) => extractText(child, acc, indexableNodeTypes)
+    );
+  }
+};
+var relativePath = (path, collection) => {
+  return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
+};
+var tokenizeString = (str) => {
+  return str.split(/[\s\.,]+/).map((s) => s.toLowerCase()).filter((s) => s);
+};
+var processTextFieldValue = (value, maxLen) => {
+  const tokens = tokenizeString(value);
+  const builder = new StringBuilder(maxLen);
+  for (const part of tokens) {
+    if (builder.append(part)) {
+      break;
+    }
+  }
+  return builder.toString();
+};
+var processDocumentForIndexing = (data, path, collection, textIndexLength, field) => {
+  if (!field) {
+    const relPath = relativePath(path, collection);
+    data["_id"] = `${collection.name}:${relPath}`;
+    data["_relativePath"] = relPath;
+  }
+  for (const f of collection.fields || (field == null ? void 0 : field.fields) || []) {
+    if (!f.searchable) {
+      delete data[f.name];
+      continue;
+    }
+    const isList = f.list;
+    if (data[f.name]) {
+      if (f.type === "object") {
+        if (isList) {
+          data[f.name] = data[f.name].map(
+            (obj) => processDocumentForIndexing(
+              obj,
+              path,
+              collection,
+              textIndexLength,
+              f
+            )
+          );
+        } else {
+          data[f.name] = processDocumentForIndexing(
+            data[f.name],
+            path,
+            collection,
+            textIndexLength,
+            f
+          );
+        }
+      } else if (f.type === "string") {
+        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+        if (isList) {
+          data[f.name] = data[f.name].map(
+            (value) => processTextFieldValue(value, fieldTextIndexLength)
+          );
+        } else {
+          data[f.name] = processTextFieldValue(
+            data[f.name],
+            fieldTextIndexLength
+          );
+        }
+      } else if (f.type === "rich-text") {
+        const fieldTextIndexLength = f.maxSearchIndexFieldLength || textIndexLength;
+        if (isList) {
+          data[f.name] = data[f.name].map((value) => {
+            const acc = new StringBuilder(fieldTextIndexLength);
+            extractText(value, acc, ["text", "code_block", "html"]);
+            return acc.toString();
+          });
+        } else {
+          const acc = new StringBuilder(fieldTextIndexLength);
+          extractText(data[f.name], acc, ["text", "code_block", "html"]);
+          data[f.name] = acc.toString();
+        }
+      }
+    }
+  }
+  return data;
+};
+
+// src/indexer/index.ts
 var SearchIndexer = class {
   constructor(options) {
     this.client = options.client;
@@ -44,65 +168,52 @@ var SearchIndexer = class {
     this.batchSize = options.batchSize || 100;
     this.textIndexLength = options.textIndexLength || 500;
   }
-  relativePath(path, collection) {
-    return path.replace(/\\/g, "/").replace(collection.path, "").replace(/^\/|\/$/g, "");
-  }
-  processDocumentForIndexing(data, path, collection, field) {
-    if (!field) {
-      data["_id"] = `${collection.name}:${this.relativePath(path, collection)}`;
-    }
-    for (const f of collection.fields || (field == null ? void 0 : field.fields) || []) {
-      const isList = f.list;
-      if (data[f.name]) {
-        if (f.type === "object") {
-          if (isList) {
-            data[f.name] = data[f.name].map(
-              (obj) => this.processDocumentForIndexing(obj, path, collection, f)
-            );
-          } else {
-            data[f.name] = this.processDocumentForIndexing(
-              data[f.name],
+  makeIndexerCallback(itemCallback) {
+    return async (collection, contentPaths) => {
+      const templateInfo = await this.schema.getTemplatesForCollectable(
+        collection
+      );
+      await (0, import_graphql.sequential)(contentPaths, async (path) => {
+        const data = await (0, import_graphql.transformDocumentIntoPayload)(
+          `${collection.path}/${path}`,
+          (0, import_graphql.transformDocument)(
+            path,
+            await (0, import_graphql.loadAndParseWithAliases)(
+              this.bridge,
               path,
               collection,
-              f
-            );
-          }
-        } else if (f.type === "image") {
-          delete data[f.name];
-        } else if (f.type === "string" || f.type === "rich-text") {
-          if (isList) {
-            data[f.name] = data[f.name].map(
-              (value) => value.substring(0, this.textIndexLength)
-            );
-          } else {
-            data[f.name] = data[f.name].substring(0, this.textIndexLength);
-          }
-        }
-      }
-    }
-    return data;
+              templateInfo
+            ),
+            this.schema
+          ),
+          this.schema
+        );
+        await itemCallback(
+          processDocumentForIndexing(
+            data["_values"],
+            path,
+            collection,
+            this.textIndexLength
+          )
+        );
+      });
+    };
   }
   async indexContentByPaths(documentPaths) {
     var _a, _b, _c, _d;
     let batch = [];
+    const itemCallback = async (item) => {
+      batch.push(item);
+      if (batch.length > this.batchSize) {
+        await this.client.put(batch);
+        batch = [];
+      }
+    };
     await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
     await (0, import_graphql.scanContentByPaths)(
       this.schema,
       documentPaths,
-      async (collection, contentPaths) => {
-        await (0, import_graphql.sequential)(contentPaths, async (path) => {
-          const data = (0, import_graphql.transformDocument)(
-            path,
-            JSON.parse(await this.bridge.get(path)),
-            this.schema
-          );
-          batch.push(this.processDocumentForIndexing(data, path, collection));
-          if (batch.length > this.batchSize) {
-            await this.client.put(batch);
-            batch = [];
-          }
-        });
-      }
+      this.makeIndexerCallback(itemCallback)
     );
     if (batch.length > 0) {
       await this.client.put(batch);
@@ -113,31 +224,17 @@ var SearchIndexer = class {
     var _a, _b, _c, _d;
     await ((_b = (_a = this.client).onStartIndexing) == null ? void 0 : _b.call(_a));
     let batch = [];
+    const itemCallback = async (item) => {
+      batch.push(item);
+      if (batch.length > this.batchSize) {
+        await this.client.put(batch);
+        batch = [];
+      }
+    };
     const warnings = await (0, import_graphql.scanAllContent)(
       this.schema,
       this.bridge,
-      async (collection, contentPaths) => {
-        const templateInfo = await this.schema.getTemplatesForCollectable(
-          collection
-        );
-        await (0, import_graphql.sequential)(contentPaths, async (path) => {
-          const data = (0, import_graphql.transformDocument)(
-            path,
-            await (0, import_graphql.loadAndParseWithAliases)(
-              this.bridge,
-              path,
-              collection,
-              templateInfo
-            ),
-            this.schema
-          );
-          batch.push(this.processDocumentForIndexing(data, path, collection));
-          if (batch.length > this.batchSize) {
-            await this.client.put(batch);
-            batch = [];
-          }
-        });
-      }
+      this.makeIndexerCallback(itemCallback)
     );
     if (batch.length > 0) {
       await this.client.put(batch);
@@ -153,216 +250,10 @@ var SearchIndexer = class {
   }
 };
 
-// src/sqlite-level.ts
-var import_abstract_level = require("abstract-level");
-var import_module_error = __toESM(require("module-error"));
-var import_better_sqlite3 = __toESM(require("better-sqlite3"));
-var queryFromOptions = (options) => {
-  let query = "SELECT key, value FROM kv";
-  const params = [];
-  if (options.gt) {
-    query += ` WHERE key > ?`;
-    params.push(options.gt);
-  } else if (options.gte) {
-    query += ` WHERE key >= ?`;
-    params.push(options.gte);
-  }
-  if (options.lt) {
-    query += ` ${options.gt || options.gte ? "AND" : "WHERE"} key < ?`;
-    params.push(options.lt);
-  } else if (options.lte) {
-    query += ` ${options.gt || options.gte ? "AND" : "WHERE"} key <= ?`;
-    params.push(options.lte);
-  }
-  if (options.reverse) {
-    query += " ORDER BY key DESC";
-  } else {
-    query += " ORDER BY key ASC";
-  }
-  if (options.limit) {
-    query += ` LIMIT ${options.limit}`;
-  }
-  return { query, params };
-};
-var SqliteIterator = class extends import_abstract_level.AbstractIterator {
-  constructor(db, options, client) {
-    super(db, options);
-    this.client = client;
-    const { query, params } = queryFromOptions(options);
-    const stmt = this.client.prepare(query);
-    this.iterator = stmt.iterate(params);
-  }
-  async _next(callback) {
-    const result = this.iterator.next();
-    if (!result.done) {
-      return this.db.nextTick(
-        callback,
-        null,
-        result.value.key,
-        result.value.value
-      );
-    } else {
-      return this.db.nextTick(callback, null, void 0, void 0);
-    }
-  }
-};
-var SqliteKeyIterator = class extends import_abstract_level.AbstractKeyIterator {
-  constructor(db, options, client) {
-    super(db, options);
-    this.client = client;
-    const { query, params } = queryFromOptions(options);
-    const stmt = this.client.prepare(query);
-    this.iterator = stmt.iterate(params);
-  }
-  async _next(callback) {
-    const result = this.iterator.next();
-    if (!result.done) {
-      return this.db.nextTick(callback, null, result.value.key);
-    } else {
-      return this.db.nextTick(callback, null, void 0);
-    }
-  }
-};
-var SqliteValueIterator = class extends import_abstract_level.AbstractValueIterator {
-  constructor(db, options, client) {
-    super(db, options);
-    this.client = client;
-    const { query, params } = queryFromOptions(options);
-    const stmt = this.client.prepare(query);
-    this.iterator = stmt.iterate(params);
-  }
-  async _next(callback) {
-    const result = this.iterator.next();
-    if (!result.done) {
-      return this.db.nextTick(callback, null, result.value.value);
-    } else {
-      return this.db.nextTick(callback, null, void 0);
-    }
-  }
-};
-var SqliteLevel = class extends import_abstract_level.AbstractLevel {
-  constructor(options) {
-    super({ encodings: { utf8: true } }, options);
-    this.readOnly = false;
-    this.db = new import_better_sqlite3.default(options.filename);
-    this.db.pragma("journal_mode = WAL");
-    if (options.readOnly !== void 0) {
-      this.readOnly = options.readOnly;
-    }
-  }
-  get type() {
-    return "sqlite3";
-  }
-  async _open(options, callback) {
-    this.db.exec("CREATE TABLE IF NOT EXISTS kv (key TEXT, value TEXT)");
-    this.nextTick(callback);
-  }
-  async _close(callback) {
-    this.db.close();
-    this.nextTick(callback);
-  }
-  async _get(key, options, callback) {
-    const stmt = this.db.prepare("SELECT value FROM kv WHERE key = ?");
-    const row = stmt.get(key.toString());
-    if (row) {
-      return this.nextTick(callback, null, row.value);
-    } else {
-      return this.nextTick(
-        callback,
-        new import_module_error.default(`Key ${key} was not found`, {
-          code: "LEVEL_NOT_FOUND"
-        })
-      );
-    }
-  }
-  async _put(key, value, options, callback) {
-    if (this.readOnly) {
-      return this.nextTick(
-        callback,
-        new import_module_error.default("not authorized to write to branch", {
-          code: "LEVEL_READ_ONLY"
-        })
-      );
-    }
-    const stmt = this.db.prepare("INSERT INTO kv (key, value) VALUES (?, ?)");
-    stmt.run(key.toString(), value.toString());
-    this.nextTick(callback);
-  }
-  async _del(key, options, callback) {
-    if (this.readOnly) {
-      return this.nextTick(
-        callback,
-        new import_module_error.default("not authorized to write to branch", {
-          code: "LEVEL_READ_ONLY"
-        })
-      );
-    }
-    const stmt = this.db.prepare("DELETE FROM kv WHERE key = ?");
-    stmt.run(key.toString());
-    this.nextTick(callback);
-  }
-  async _batch(batch, options, callback) {
-    if (this.readOnly) {
-      return this.nextTick(
-        callback,
-        new import_module_error.default("not authorized to write to branch", {
-          code: "LEVEL_READ_ONLY"
-        })
-      );
-    }
-    let batches = [];
-    let curBatch = [];
-    let curType = void 0;
-    for (const op of batch) {
-      if (curType === void 0) {
-        curType = op.type;
-      } else if (curType !== op.type) {
-        if (curType === "put") {
-          batches.push(
-            `INSERT INTO kv (key, value) VALUES ${curBatch.join(",")}`
-          );
-        } else if (curType === "del") {
-          batches.push(`DELETE FROM kv WHERE key IN (${curBatch.join(",")})`);
-        }
-        curBatch = [];
-        curType = op.type;
-      }
-      if (op.type === "put") {
-        curBatch.push(`('${op.key.toString()}', '${op.value.toString()}')`);
-      } else if (op.type === "del") {
-        curBatch.push(`'${op.key.toString()}'`);
-      }
-    }
-    if (curBatch.length > 0) {
-      if (curType === "put") {
-        batches.push(`INSERT INTO kv (key, value) VALUES ${curBatch.join(",")}`);
-      } else if (curType === "del") {
-        batches.push(`DELETE FROM kv WHERE key IN (${curBatch.join(",")})`);
-      }
-    }
-    for (const batch2 of batches) {
-      this.db.exec(batch2);
-    }
-    this.nextTick(callback);
-  }
-  async _clear(options, callback) {
-    this.db.exec(`DELETE FROM kv WHERE key like '${options.gte}%'`);
-    this.nextTick(callback);
-  }
-  _iterator(options) {
-    return new SqliteIterator(this, options, this.db);
-  }
-  _keys(options) {
-    return new SqliteKeyIterator(this, options, this.db);
-  }
-  _values(options) {
-    return new SqliteValueIterator(this, options, this.db);
-  }
-};
-
 // src/client/index.ts
+var import_sqlite_level = require("sqlite-level");
 var zlib = __toESM(require("zlib"));
-var import_search_index = __toESM(require("@kldavis4/search-index"));
+var import_search_index = __toESM(require("@tinacms/search-index"));
 var import_memory_level = require("memory-level");
 var import_node_fetch = __toESM(require("node-fetch"));
 var LocalSearchIndexClient = class {
@@ -412,26 +303,39 @@ var TinaCMSSearchIndexClient = class {
     const headers = new import_node_fetch.Headers();
     headers.append("x-api-key", this.indexerToken || "bogus");
     headers.append("Content-Type", "application/json");
-    const res = await (0, import_node_fetch.default)(`${this.apiUrl}/upload/${this.branch}`, {
+    let res = await (0, import_node_fetch.default)(`${this.apiUrl}/upload/${this.branch}`, {
       method: "GET",
       headers
     });
+    if (res.status !== 200) {
+      let json;
+      try {
+        json = await res.json();
+      } catch (e) {
+        console.error("Failed to parse error response", e);
+      }
+      throw new Error(
+        `Failed to get upload url. Status: ${res.status}${(json == null ? void 0 : json.message) ? ` - ${json.message}` : ``}`
+      );
+    }
     const { signedUrl } = await res.json();
-    console.log("onFinishIndexing");
-    const sqliteLevel = new SqliteLevel({ filename: ":memory:" });
-    console.log("dumping to sqlite");
+    const sqliteLevel = new import_sqlite_level.SqliteLevel({ filename: ":memory:" });
     const iterator = this.memoryLevel.iterator();
     for await (const [key, value] of iterator) {
       await sqliteLevel.put(key, value);
     }
     const buffer = sqliteLevel.db.serialize();
-    console.log(buffer.byteLength);
     await sqliteLevel.close();
-    await (0, import_node_fetch.default)(signedUrl, {
+    res = await (0, import_node_fetch.default)(signedUrl, {
       method: "PUT",
       body: zlib.gzipSync(buffer)
     });
-    console.log("done onFinishIndexing");
+    if (res.status !== 200) {
+      throw new Error(
+        `Failed to upload search index. Status: ${res.status}
+${await res.text()}`
+      );
+    }
   }
   getSearchIndex() {
     return this.searchIndex;
@@ -464,5 +368,6 @@ var lookupStopwords = (keys, defaultStopWords = sw.eng) => {
   SearchIndexer,
   TinaCMSSearchIndexClient,
   lookupStopwords,
+  si,
   sw
 });

package/dist/indexer/index.d.ts

@@ -15,8 +15,7 @@ export declare class SearchIndexer {
     private readonly schema;
     private readonly textIndexLength;
     constructor(options: SearchIndexOptions);
-    private relativePath;
-    private processDocumentForIndexing;
+    private makeIndexerCallback;
     indexContentByPaths(documentPaths: string[]): Promise<void>;
     indexAllContent(): Promise<{
         warnings: string[];

package/dist/indexer/utils.d.ts

@@ -0,0 +1,2 @@
+import { Collection, ObjectField } from '@tinacms/schema-tools';
+export declare const processDocumentForIndexing: (data: any, path: string, collection: Collection, textIndexLength: number, field?: ObjectField) => any;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tinacms/search",
-  "version": "1.0.0",
+  "version": "1.0.2",
   "main": "dist/index.js",
   "module": "dist/index-client.es.js",
   "typings": "dist/index.d.ts",
@@ -9,8 +9,14 @@
     "dist"
   ],
   "exports": {
-    "import": "./dist/index-client.es.js",
-    "require": "./dist/index.js"
+    ".": {
+      "import": "./dist/index.es.js",
+      "require": "./dist/index.js"
+    },
+    "./dist/index-client": {
+      "import": "./dist/index-client.es.js",
+      "require": "./dist/index-client.js"
+    }
   },
   "license": "SEE LICENSE IN LICENSE",
   "buildConfig": {
@@ -23,16 +29,15 @@
     ]
   },
   "dependencies": {
-    "@tinacms/graphql": "1.4.14",
-    "@tinacms/schema-tools": "1.4.4",
+    "@tinacms/graphql": "1.4.17",
+    "@tinacms/schema-tools": "1.4.6",
     "abstract-level": "^1.0.3",
-    "better-sqlite3": "^7.4.2",
-    "classic-level": "^1.3.0",
     "memory-level": "^1.0.0",
     "module-error": "^1.0.2",
     "node-fetch": "2",
-    "@kldavis4/search-index": "4.4.1",
-    "stopword": "^2.0.8"
+    "@tinacms/search-index": "^3.5.2",
+    "stopword": "^2.0.8",
+    "sqlite-level": "^1.0.1"
   },
   "publishConfig": {
     "registry": "https://registry.npmjs.org"

package/dist/sqlite-level.d.ts

@@ -1,97 +0,0 @@
-/// <reference types="node" />
-/**
- Copyright 2023 Forestry.io Holdings, Inc.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-import { AbstractDatabaseOptions, AbstractIterator, AbstractKeyIterator, AbstractLevel, AbstractOpenOptions, AbstractValueIterator } from 'abstract-level';
-import { NextCallback } from 'abstract-level/types/abstract-iterator';
-export declare type SqliteLevelOptions<K, V> = {
-    filename: string;
-    readOnly?: boolean;
-} & AbstractDatabaseOptions<K, V>;
-declare type BatchOperation = BatchPutOperation | BatchDelOperation;
-/**
- * A _put_ operation to be committed by a {@link SqliteLevel}.
- */
-declare interface BatchPutOperation {
-    /**
-     * Type of operation.
-     */
-    type: 'put';
-    /**
-     * Key of the entry to be added to the database.
-     */
-    key: Buffer;
-    /**
-     * Value of the entry to be added to the database.
-     */
-    value: Buffer;
-}
-/**
- * A _del_ operation to be committed by a {@link SqliteLevel}.
- */
-declare interface BatchDelOperation {
-    /**
-     * Type of operation.
-     */
-    type: 'del';
-    /**
-     * Key of the entry to be deleted from the database.
-     */
-    key: Buffer;
-}
-declare interface IteratorOptions<KDefault> {
-    limit?: number;
-    keyEncoding: string;
-    valueEncoding: string;
-    reverse: boolean;
-    keys: boolean;
-    values: boolean;
-    gt?: KDefault;
-    gte?: KDefault;
-    lt?: KDefault;
-    lte?: KDefault;
-}
-declare class SqliteIterator<KDefault, VDefault> extends AbstractIterator<SqliteLevel<KDefault, VDefault>, KDefault, VDefault> {
-    private client;
-    private iterator;
-    constructor(db: SqliteLevel<KDefault, VDefault>, options: IteratorOptions<KDefault>, client: any);
-    _next(callback: NextCallback<KDefault, VDefault>): Promise<void>;
-}
-declare class SqliteKeyIterator<KDefault, VDefault> extends AbstractKeyIterator<SqliteLevel<KDefault, VDefault>, KDefault> {
-    private client;
-    private iterator;
-    constructor(db: SqliteLevel<KDefault, VDefault>, options: IteratorOptions<KDefault>, client: any);
-    _next(callback: NextCallback<KDefault, VDefault>): Promise<void>;
-}
-declare class SqliteValueIterator<KDefault, VDefault> extends AbstractValueIterator<SqliteLevel<KDefault, VDefault>, KDefault, VDefault> {
-    private client;
-    private iterator;
-    constructor(db: SqliteLevel<KDefault, VDefault>, options: IteratorOptions<KDefault>, client: any);
-    _next(callback: NextCallback<KDefault, VDefault>): Promise<void>;
-}
-export declare class SqliteLevel<KDefault = string, VDefault = string> extends AbstractLevel<Buffer | Uint8Array | string, KDefault, VDefault> {
-    db: any;
-    private readOnly;
-    constructor(options: SqliteLevelOptions<KDefault, VDefault>);
-    get type(): string;
-    _open(options: AbstractOpenOptions, callback: (error?: Error) => void): Promise<void>;
-    _close(callback: (error?: Error) => void): Promise<void>;
-    _get(key: Buffer, options: any, callback: (error?: Error, value?: Buffer) => void): Promise<void>;
-    _put(key: Buffer, value: Buffer, options: any, callback: (error?: Error) => void): Promise<void>;
-    _del(key: Buffer, options: any, callback: (error?: Error) => void): Promise<void>;
-    _batch(batch: BatchOperation[], options: any, callback: (error?: Error) => void): Promise<void>;
-    _clear(options: any, callback: (error?: Error) => void): Promise<void>;
-    _iterator(options: IteratorOptions<KDefault>): SqliteIterator<KDefault, VDefault>;
-    _keys(options: IteratorOptions<KDefault>): SqliteKeyIterator<KDefault, VDefault>;
-    _values(options: IteratorOptions<KDefault>): SqliteValueIterator<KDefault, VDefault>;
-}
-export {};