@peam-ai/search 0.1.1

package/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/README.md

@@ -0,0 +1,11 @@
+# @peam-ai/search
+
+Vector database and search functionality for Peam.
+
+## Documentation
+
+For detailed documentation, visit [peam.ai/docs](https://peam.ai/docs)
+
+## License
+
+Apache-2.0

package/dist/index.d.mts

@@ -0,0 +1,66 @@
+import { StructuredPage } from '@peam-ai/parser';
+
+interface PageToIndex {
+    path: string;
+    structuredPage: StructuredPage;
+}
+interface SearchIndexData {
+    keys: string[];
+    data: Record<string, string>;
+}
+/**
+ * Build a search index from structured pages
+ */
+declare function buildSearchIndex(pages: PageToIndex[]): Promise<SearchIndexData>;
+
+type StructuredPageDocumentData = {
+    id: string;
+    path: string;
+    content: StructuredPage;
+    [key: string]: any;
+};
+
+interface TextSearchOptions {
+    limit?: number;
+    offset?: number;
+    suggest?: boolean;
+}
+declare class TextSearch {
+    private index;
+    private initialized;
+    private documentIds;
+    constructor();
+    private getIndex;
+    initialize(): Promise<void>;
+    addDocument(document: StructuredPageDocumentData): Promise<void>;
+    search(query: string, options?: TextSearchOptions): Promise<StructuredPageDocumentData[]>;
+    count(): number;
+    getDocument(path: string): StructuredPageDocumentData | null;
+    getAllDocuments(limit?: number): StructuredPageDocumentData[];
+    clear(): void;
+    export(handler: (key: string, data: string) => Promise<void>): Promise<{
+        keys: string[];
+    }>;
+    import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void>;
+}
+
+interface SearchEngineConfig {
+}
+declare class SearchEngine {
+    private textSearch;
+    private initialized;
+    constructor(_config?: SearchEngineConfig);
+    initialize(): Promise<void>;
+    addPage(path: string, content: StructuredPage): Promise<void>;
+    search(query: string, options?: TextSearchOptions): Promise<StructuredPageDocumentData[]>;
+    count(): number;
+    getDocument(path: string): StructuredPageDocumentData | null;
+    getAllDocuments(limit?: number): StructuredPageDocumentData[];
+    clear(): void;
+    export(handler: (key: string, data: string) => Promise<void>): Promise<{
+        keys: string[];
+    }>;
+    import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void>;
+}
+
+export { type PageToIndex, SearchEngine, type SearchEngineConfig, type SearchIndexData, type StructuredPageDocumentData, TextSearch, buildSearchIndex };

package/dist/index.d.ts

@@ -0,0 +1,66 @@
+import { StructuredPage } from '@peam-ai/parser';
+
+interface PageToIndex {
+    path: string;
+    structuredPage: StructuredPage;
+}
+interface SearchIndexData {
+    keys: string[];
+    data: Record<string, string>;
+}
+/**
+ * Build a search index from structured pages
+ */
+declare function buildSearchIndex(pages: PageToIndex[]): Promise<SearchIndexData>;
+
+type StructuredPageDocumentData = {
+    id: string;
+    path: string;
+    content: StructuredPage;
+    [key: string]: any;
+};
+
+interface TextSearchOptions {
+    limit?: number;
+    offset?: number;
+    suggest?: boolean;
+}
+declare class TextSearch {
+    private index;
+    private initialized;
+    private documentIds;
+    constructor();
+    private getIndex;
+    initialize(): Promise<void>;
+    addDocument(document: StructuredPageDocumentData): Promise<void>;
+    search(query: string, options?: TextSearchOptions): Promise<StructuredPageDocumentData[]>;
+    count(): number;
+    getDocument(path: string): StructuredPageDocumentData | null;
+    getAllDocuments(limit?: number): StructuredPageDocumentData[];
+    clear(): void;
+    export(handler: (key: string, data: string) => Promise<void>): Promise<{
+        keys: string[];
+    }>;
+    import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void>;
+}
+
+interface SearchEngineConfig {
+}
+declare class SearchEngine {
+    private textSearch;
+    private initialized;
+    constructor(_config?: SearchEngineConfig);
+    initialize(): Promise<void>;
+    addPage(path: string, content: StructuredPage): Promise<void>;
+    search(query: string, options?: TextSearchOptions): Promise<StructuredPageDocumentData[]>;
+    count(): number;
+    getDocument(path: string): StructuredPageDocumentData | null;
+    getAllDocuments(limit?: number): StructuredPageDocumentData[];
+    clear(): void;
+    export(handler: (key: string, data: string) => Promise<void>): Promise<{
+        keys: string[];
+    }>;
+    import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void>;
+}
+
+export { type PageToIndex, SearchEngine, type SearchEngineConfig, type SearchIndexData, type StructuredPageDocumentData, TextSearch, buildSearchIndex };

package/dist/index.js

@@ -0,0 +1,291 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var __async = (__this, __arguments, generator) => {
+  return new Promise((resolve, reject) => {
+    var fulfilled = (value) => {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var rejected = (value) => {
+      try {
+        step(generator.throw(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
+    step((generator = generator.apply(__this, __arguments)).next());
+  });
+};
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  SearchEngine: () => SearchEngine,
+  TextSearch: () => TextSearch,
+  buildSearchIndex: () => buildSearchIndex
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/searchEngine.ts
+var import_logger2 = require("@peam-ai/logger");
+
+// src/textSearch.ts
+var import_logger = require("@peam-ai/logger");
+var import_flexsearch = require("flexsearch");
+var PEAM_DOCUMENT_IDS_KEY = "peam.documentIds";
+var MAX_DOCUMENTS_RETRIEVE = 25;
+var log = import_logger.loggers.search;
+var TextSearch = class {
+  constructor() {
+    this.initialized = false;
+    this.index = this.getIndex();
+    this.documentIds = /* @__PURE__ */ new Set();
+  }
+  getIndex() {
+    return new import_flexsearch.Document({
+      worker: false,
+      document: {
+        id: "path",
+        index: ["content:title", "content:description", "content:textContent", "content:author", "content:keywords"],
+        store: true
+      },
+      tokenize: "forward",
+      resolution: 9,
+      context: {
+        resolution: 3,
+        depth: 2,
+        bidirectional: true
+      },
+      cache: 100,
+      encoder: import_flexsearch.Charset.LatinExtra
+    });
+  }
+  initialize() {
+    return __async(this, null, function* () {
+      if (this.initialized) {
+        log.debug("Text search already initialized");
+        return;
+      }
+      this.initialized = true;
+    });
+  }
+  addDocument(document) {
+    return __async(this, null, function* () {
+      if (!this.initialized) {
+        throw new Error("TextSearch not initialized. Call initialize() first.");
+      }
+      log.debug("Adding document to text search:", document.path);
+      this.index.add(document);
+      this.documentIds.add(document.path);
+    });
+  }
+  search(_0) {
+    return __async(this, arguments, function* (query, options = {}) {
+      if (!this.initialized) {
+        throw new Error("TextSearch not initialized. Call initialize() first.");
+      }
+      const limit = options.limit || MAX_DOCUMENTS_RETRIEVE;
+      const offset = options.offset || 0;
+      log.debug("Searching for:", query);
+      const results = yield this.index.search(query, {
+        limit: limit + offset,
+        suggest: options.suggest,
+        enrich: true
+      });
+      const pathSet = /* @__PURE__ */ new Set();
+      const documents = [];
+      for (const fieldResults of results) {
+        if (Array.isArray(fieldResults.result)) {
+          for (const result of fieldResults.result) {
+            const id = typeof result === "object" && "id" in result ? result.id : result;
+            const doc = typeof result === "object" && "doc" in result ? result.doc : null;
+            if (!pathSet.has(id) && doc) {
+              pathSet.add(id);
+              documents.push(doc);
+            }
+          }
+        }
+      }
+      const pagedResults = documents.slice(offset, offset + limit);
+      return pagedResults;
+    });
+  }
+  count() {
+    return this.documentIds.size;
+  }
+  getDocument(path) {
+    return this.index.get(path);
+  }
+  getAllDocuments(limit) {
+    const documents = [];
+    let count = 0;
+    limit = limit || MAX_DOCUMENTS_RETRIEVE;
+    for (const id of this.documentIds) {
+      if (count >= limit) {
+        break;
+      }
+      const doc = this.index.get(id);
+      if (doc) {
+        documents.push(doc);
+        count++;
+      }
+    }
+    log.debug("Retrieved documents from store (limit):", documents.length, limit);
+    return documents;
+  }
+  clear() {
+    this.index.clear();
+    this.index = this.getIndex();
+    this.documentIds.clear();
+  }
+  export(handler) {
+    return __async(this, null, function* () {
+      const keys = [];
+      yield handler(PEAM_DOCUMENT_IDS_KEY, JSON.stringify(Array.from(this.documentIds)));
+      keys.push(PEAM_DOCUMENT_IDS_KEY);
+      yield this.index.export((key, data) => __async(this, null, function* () {
+        keys.push(key);
+        yield handler(key, data);
+      }));
+      log.debug("Exported keys:", keys.length);
+      return { keys };
+    });
+  }
+  import(handler, keys) {
+    return __async(this, null, function* () {
+      const documentIdsData = yield handler(PEAM_DOCUMENT_IDS_KEY);
+      if (documentIdsData) {
+        const parsed = typeof documentIdsData === "string" ? JSON.parse(documentIdsData) : documentIdsData;
+        this.documentIds = new Set(parsed);
+      }
+      for (const key of keys) {
+        if (key === PEAM_DOCUMENT_IDS_KEY) {
+          continue;
+        }
+        try {
+          const data = yield handler(key);
+          if (data) {
+            this.index.import(key, data);
+          }
+        } catch (error) {
+          log.error("Error importing key:", key, error);
+        }
+      }
+      this.initialized = true;
+      log.debug("Import completed with keys:", keys.length);
+    });
+  }
+};
+
+// src/searchEngine.ts
+var log2 = import_logger2.loggers.search;
+var SearchEngine = class {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  constructor(_config) {
+    this.textSearch = new TextSearch();
+    this.initialized = false;
+  }
+  initialize() {
+    return __async(this, null, function* () {
+      if (this.initialized) {
+        return;
+      }
+      log2.debug("Initializing search engine");
+      yield this.textSearch.initialize();
+      this.initialized = true;
+    });
+  }
+  addPage(path, content) {
+    return __async(this, null, function* () {
+      if (!this.initialized) {
+        throw new Error("Search engine not initialized. Call initialize() first.");
+      }
+      const document = {
+        id: path,
+        path,
+        content
+      };
+      yield this.textSearch.addDocument(document);
+      log2.debug("Page added to search engine:", path);
+    });
+  }
+  search(_0) {
+    return __async(this, arguments, function* (query, options = {}) {
+      if (!this.initialized) {
+        throw new Error("Search engine not initialized. Call initialize() first.");
+      }
+      log2.debug("Performing text search:", query);
+      return this.textSearch.search(query, options);
+    });
+  }
+  count() {
+    return this.textSearch.count();
+  }
+  getDocument(path) {
+    return this.textSearch.getDocument(path);
+  }
+  getAllDocuments(limit) {
+    return this.textSearch.getAllDocuments(limit);
+  }
+  clear() {
+    this.textSearch.clear();
+  }
+  export(handler) {
+    return __async(this, null, function* () {
+      return this.textSearch.export(handler);
+    });
+  }
+  import(handler, keys) {
+    return __async(this, null, function* () {
+      yield this.textSearch.import(handler, keys);
+      this.initialized = true;
+      log2.debug("Search engine initialized from imported data");
+    });
+  }
+};
+
+// src/indexBuilder.ts
+function buildSearchIndex(pages) {
+  return __async(this, null, function* () {
+    const searchEngine = new SearchEngine();
+    yield searchEngine.initialize();
+    for (const page of pages) {
+      yield searchEngine.addPage(page.path, page.structuredPage);
+    }
+    const exportedData = {};
+    const result = yield searchEngine.export((key, data) => __async(null, null, function* () {
+      exportedData[key] = data;
+    }));
+    return {
+      keys: result.keys,
+      data: exportedData
+    };
+  });
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  SearchEngine,
+  TextSearch,
+  buildSearchIndex
+});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/searchEngine.ts","../src/textSearch.ts","../src/indexBuilder.ts"],"sourcesContent":["export { buildSearchIndex, type PageToIndex, type SearchIndexData } from './indexBuilder';\nexport { SearchEngine } from './searchEngine';\nexport { TextSearch } from './textSearch';\n\nexport type { StructuredPageDocumentData } from './types';\n\nexport type { SearchEngineConfig } from './searchEngine';\n","import { loggers } from '@peam-ai/logger';\nimport type { StructuredPage } from '@peam-ai/parser';\nimport { TextSearch, type TextSearchOptions } from './textSearch';\nimport type { StructuredPageDocumentData } from './types';\n\nconst log = loggers.search;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-object-type\nexport interface SearchEngineConfig {\n  // Reserved for future configuration options\n}\n\nexport class SearchEngine {\n  private textSearch: TextSearch;\n  private initialized: boolean;\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  constructor(_config?: SearchEngineConfig) {\n    this.textSearch = new TextSearch();\n    this.initialized = false;\n  }\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      return;\n    }\n\n    log.debug('Initializing search engine');\n    await this.textSearch.initialize();\n    this.initialized = true;\n  }\n\n  async addPage(path: string, content: StructuredPage): Promise<void> {\n    if (!this.initialized) {\n      throw new Error('Search engine not initialized. Call initialize() first.');\n    }\n\n    const document: StructuredPageDocumentData = {\n      id: path,\n      path,\n      content,\n    };\n\n    await this.textSearch.addDocument(document);\n    log.debug('Page added to search engine:', path);\n  }\n\n  async search(query: string, options: TextSearchOptions = {}): Promise<StructuredPageDocumentData[]> {\n    if (!this.initialized) {\n      throw new Error('Search engine not initialized. Call initialize() first.');\n    }\n\n    log.debug('Performing text search:', query);\n    return this.textSearch.search(query, options);\n  }\n\n  count(): number {\n    return this.textSearch.count();\n  }\n\n  getDocument(path: string) {\n    return this.textSearch.getDocument(path);\n  }\n\n  getAllDocuments(limit?: number): StructuredPageDocumentData[] {\n    return this.textSearch.getAllDocuments(limit);\n  }\n\n  clear(): void {\n    this.textSearch.clear();\n  }\n\n  async export(handler: (key: string, data: string) => Promise<void>): Promise<{ keys: string[] }> {\n    return this.textSearch.export(handler);\n  }\n\n  async import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void> {\n    await this.textSearch.import(handler, keys);\n    this.initialized = true;\n    log.debug('Search engine initialized from imported data');\n  }\n}\n","import { loggers } from '@peam-ai/logger';\nimport { Charset, Document } from 'flexsearch';\nimport type { StructuredPageDocumentData } from './types';\n\nexport interface TextSearchOptions {\n  limit?: number;\n  offset?: number;\n  suggest?: boolean;\n}\n\nconst PEAM_DOCUMENT_IDS_KEY = 'peam.documentIds';\nconst MAX_DOCUMENTS_RETRIEVE = 25;\nconst log = loggers.search;\n\nexport class TextSearch {\n  private index: Document<StructuredPageDocumentData>;\n  private initialized: boolean;\n  private documentIds: Set<string>;\n\n  constructor() {\n    this.initialized = false;\n    this.index = this.getIndex();\n    this.documentIds = new Set();\n  }\n\n  private getIndex() {\n    return new Document<StructuredPageDocumentData>({\n      worker: false,\n      document: {\n        id: 'path',\n        index: ['content:title', 'content:description', 'content:textContent', 'content:author', 'content:keywords'],\n        store: true,\n      },\n      tokenize: 'forward',\n      resolution: 9,\n      context: {\n        resolution: 3,\n        depth: 2,\n        bidirectional: true,\n      },\n      cache: 100,\n      encoder: Charset.LatinExtra,\n    });\n  }\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      log.debug('Text search already initialized');\n      return;\n    }\n\n    this.initialized = true;\n  }\n\n  async addDocument(document: StructuredPageDocumentData): Promise<void> {\n    if (!this.initialized) {\n      throw new Error('TextSearch not initialized. Call initialize() first.');\n    }\n\n    log.debug('Adding document to text search:', document.path);\n\n    this.index.add(document);\n    this.documentIds.add(document.path);\n  }\n\n  async search(query: string, options: TextSearchOptions = {}): Promise<StructuredPageDocumentData[]> {\n    if (!this.initialized) {\n      throw new Error('TextSearch not initialized. Call initialize() first.');\n    }\n\n    const limit = options.limit || MAX_DOCUMENTS_RETRIEVE;\n    const offset = options.offset || 0;\n\n    log.debug('Searching for:', query);\n\n    const results = await this.index.search(query, {\n      limit: limit + offset,\n      suggest: options.suggest,\n      enrich: true,\n    });\n\n    const pathSet = new Set<string>();\n    const documents: StructuredPageDocumentData[] = [];\n\n    for (const fieldResults of results) {\n      if (Array.isArray(fieldResults.result)) {\n        for (const result of fieldResults.result) {\n          const id = typeof result === 'object' && 'id' in result ? result.id : result;\n          const doc = typeof result === 'object' && 'doc' in result ? result.doc : null;\n\n          if (!pathSet.has(id as string) && doc) {\n            pathSet.add(id as string);\n            documents.push(doc);\n          }\n        }\n      }\n    }\n\n    const pagedResults = documents.slice(offset, offset + limit);\n\n    return pagedResults;\n  }\n\n  count(): number {\n    return this.documentIds.size;\n  }\n\n  getDocument(path: string) {\n    return this.index.get(path);\n  }\n\n  getAllDocuments(limit?: number): StructuredPageDocumentData[] {\n    const documents: StructuredPageDocumentData[] = [];\n    let count = 0;\n    limit = limit || MAX_DOCUMENTS_RETRIEVE;\n\n    for (const id of this.documentIds) {\n      if (count >= limit) {\n        break;\n      }\n\n      const doc = this.index.get(id);\n      if (doc) {\n        documents.push(doc);\n        count++;\n      }\n    }\n\n    log.debug('Retrieved documents from store (limit):', documents.length, limit);\n    return documents;\n  }\n\n  clear(): void {\n    this.index.clear();\n    this.index = this.getIndex();\n    this.documentIds.clear();\n  }\n\n  async export(handler: (key: string, data: string) => Promise<void>): Promise<{ keys: string[] }> {\n    const keys: string[] = [];\n\n    await handler(PEAM_DOCUMENT_IDS_KEY, JSON.stringify(Array.from(this.documentIds)));\n    keys.push(PEAM_DOCUMENT_IDS_KEY);\n\n    await this.index.export(async (key: string, data: string) => {\n      keys.push(key);\n      await handler(key, data);\n    });\n\n    log.debug('Exported keys:', keys.length);\n\n    return { keys };\n  }\n\n  async import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void> {\n    const documentIdsData = await handler(PEAM_DOCUMENT_IDS_KEY);\n    if (documentIdsData) {\n      const parsed = typeof documentIdsData === 'string' ? JSON.parse(documentIdsData) : documentIdsData;\n      this.documentIds = new Set(parsed);\n    }\n\n    for (const key of keys) {\n      if (key === PEAM_DOCUMENT_IDS_KEY) {\n        continue;\n      }\n\n      try {\n        const data = await handler(key);\n        if (data) {\n          this.index.import(key, data);\n        }\n      } catch (error) {\n        log.error('Error importing key:', key, error);\n      }\n    }\n\n    this.initialized = true;\n    log.debug('Import completed with keys:', keys.length);\n  }\n}\n","import type { StructuredPage } from '@peam-ai/parser';\nimport { SearchEngine } from './searchEngine';\n\nexport interface PageToIndex {\n  path: string;\n  structuredPage: StructuredPage;\n}\n\nexport interface SearchIndexData {\n  keys: string[];\n  data: Record<string, string>;\n}\n\n/**\n * Build a search index from structured pages\n */\nexport async function buildSearchIndex(pages: PageToIndex[]): Promise<SearchIndexData> {\n  const searchEngine = new SearchEngine();\n  await searchEngine.initialize();\n\n  for (const page of pages) {\n    await searchEngine.addPage(page.path, page.structuredPage);\n  }\n\n  const exportedData: Record<string, string> = {};\n  const result = await searchEngine.export(async (key, data) => {\n    exportedData[key] = data;\n  });\n\n  return {\n    keys: result.keys,\n    data: exportedData,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACAA,IAAAA,iBAAwB;;;ACAxB,oBAAwB;AACxB,wBAAkC;AASlC,IAAM,wBAAwB;AAC9B,IAAM,yBAAyB;AAC/B,IAAM,MAAM,sBAAQ;AAEb,IAAM,aAAN,MAAiB;AAAA,EAKtB,cAAc;AACZ,SAAK,cAAc;AACnB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,cAAc,oBAAI,IAAI;AAAA,EAC7B;AAAA,EAEQ,WAAW;AACjB,WAAO,IAAI,2BAAqC;AAAA,MAC9C,QAAQ;AAAA,MACR,UAAU;AAAA,QACR,IAAI;AAAA,QACJ,OAAO,CAAC,iBAAiB,uBAAuB,uBAAuB,kBAAkB,kBAAkB;AAAA,QAC3G,OAAO;AAAA,MACT;AAAA,MACA,UAAU;AAAA,MACV,YAAY;AAAA,MACZ,SAAS;AAAA,QACP,YAAY;AAAA,QACZ,OAAO;AAAA,QACP,eAAe;AAAA,MACjB;AAAA,MACA,OAAO;AAAA,MACP,SAAS,0BAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEM,aAA4B;AAAA;AAChC,UAAI,KAAK,aAAa;AACpB,YAAI,MAAM,iCAAiC;AAC3C;AAAA,MACF;AAEA,WAAK,cAAc;AAAA,IACrB;AAAA;AAAA,EAEM,YAAY,UAAqD;AAAA;AACrE,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,sDAAsD;AAAA,MACxE;AAEA,UAAI,MAAM,mCAAmC,SAAS,IAAI;AAE1D,WAAK,MAAM,IAAI,QAAQ;AACvB,WAAK,YAAY,IAAI,SAAS,IAAI;AAAA,IACpC;AAAA;AAAA,EAEM,OAAO,IAAuF;AAAA,+CAAvF,OAAe,UAA6B,CAAC,GAA0C;AAClG,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,sDAAsD;AAAA,MACxE;AAEA,YAAM,QAAQ,QAAQ,SAAS;AAC/B,YAAM,SAAS,QAAQ,UAAU;AAEjC,UAAI,MAAM,kBAAkB,KAAK;AAEjC,YAAM,UAAU,MAAM,KAAK,MAAM,OAAO,OAAO;AAAA,QAC7C,OAAO,QAAQ;AAAA,QACf,SAAS,QAAQ;AAAA,QACjB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,YAA0C,CAAC;AAEjD,iBAAW,gBAAgB,SAAS;AAClC,YAAI,MAAM,QAAQ,aAAa,MAAM,GAAG;AACtC,qBAAW,UAAU,aAAa,QAAQ;AACxC,kBAAM,KAAK,OAAO,WAAW,YAAY,QAAQ,SAAS,OAAO,KAAK;AACtE,kBAAM,MAAM,OAAO,WAAW,YAAY,SAAS,SAAS,OAAO,MAAM;AAEzE,gBAAI,CAAC,QAAQ,IAAI,EAAY,KAAK,KAAK;AACrC,sBAAQ,IAAI,EAAY;AACxB,wBAAU,KAAK,GAAG;AAAA,YACpB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,YAAM,eAAe,UAAU,MAAM,QAAQ,SAAS,KAAK;AAE3D,aAAO;AAAA,IACT;AAAA;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,YAAY,MAAc;AACxB,WAAO,KAAK,MAAM,IAAI,IAAI;AAAA,EAC5B;AAAA,EAEA,gBAAgB,OAA8C;AAC5D,UAAM,YAA0C,CAAC;AACjD,QAAI,QAAQ;AACZ,YAAQ,SAAS;AAEjB,eAAW,MAAM,KAAK,aAAa;AACjC,UAAI,SAAS,OAAO;AAClB;AAAA,MACF;AAEA,YAAM,MAAM,KAAK,MAAM,IAAI,EAAE;AAC7B,UAAI,KAAK;AACP,kBAAU,KAAK,GAAG;AAClB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,MAAM,2CAA2C,UAAU,QAAQ,KAAK;AAC5E,WAAO;AAAA,EACT;AAAA,EAEA,QAAc;AACZ,SAAK,MAAM,MAAM;AACjB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA,EAEM,OAAO,SAAoF;AAAA;AAC/F,YAAM,OAAiB,CAAC;AAExB,YAAM,QAAQ,uBAAuB,KAAK,UAAU,MAAM,KAAK,KAAK,WAAW,CAAC,CAAC;AACjF,WAAK,KAAK,qBAAqB;AAE/B,YAAM,KAAK,MAAM,OAAO,CAAO,KAAa,SAAiB;AAC3D,aAAK,KAAK,GAAG;AACb,cAAM,QAAQ,KAAK,IAAI;AAAA,MACzB,EAAC;AAED,UAAI,MAAM,kBAAkB,KAAK,MAAM;AAEvC,aAAO,EAAE,KAAK;AAAA,IAChB;AAAA;AAAA,EAEM,OAAO,SAA2C,MAA+B;AAAA;AACrF,YAAM,kBAAkB,MAAM,QAAQ,qBAAqB;AAC3D,UAAI,iBAAiB;AACnB,cAAM,SAAS,OAAO,oBAAoB,WAAW,KAAK,MAAM,eAAe,IAAI;AACnF,aAAK,cAAc,IAAI,IAAI,MAAM;AAAA,MACnC;AAEA,iBAAW,OAAO,MAAM;AACtB,YAAI,QAAQ,uBAAuB;AACjC;AAAA,QACF;AAEA,YAAI;AACF,gBAAM,OAAO,MAAM,QAAQ,GAAG;AAC9B,cAAI,MAAM;AACR,iBAAK,MAAM,OAAO,KAAK,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,OAAO;AACd,cAAI,MAAM,wBAAwB,KAAK,KAAK;AAAA,QAC9C;AAAA,MACF;AAEA,WAAK,cAAc;AACnB,UAAI,MAAM,+BAA+B,KAAK,MAAM;AAAA,IACtD;AAAA;AACF;;;AD9KA,IAAMC,OAAM,uBAAQ;AAOb,IAAM,eAAN,MAAmB;AAAA;AAAA,EAKxB,YAAY,SAA8B;AACxC,SAAK,aAAa,IAAI,WAAW;AACjC,SAAK,cAAc;AAAA,EACrB;AAAA,EAEM,aAA4B;AAAA;AAChC,UAAI,KAAK,aAAa;AACpB;AAAA,MACF;AAEA,MAAAA,KAAI,MAAM,4BAA4B;AACtC,YAAM,KAAK,WAAW,WAAW;AACjC,WAAK,cAAc;AAAA,IACrB;AAAA;AAAA,EAEM,QAAQ,MAAc,SAAwC;AAAA;AAClE,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,yDAAyD;AAAA,MAC3E;AAEA,YAAM,WAAuC;AAAA,QAC3C,IAAI;AAAA,QACJ;AAAA,QACA;AAAA,MACF;AAEA,YAAM,KAAK,WAAW,YAAY,QAAQ;AAC1C,MAAAA,KAAI,MAAM,gCAAgC,IAAI;AAAA,IAChD;AAAA;AAAA,EAEM,OAAO,IAAuF;AAAA,+CAAvF,OAAe,UAA6B,CAAC,GAA0C;AAClG,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,yDAAyD;AAAA,MAC3E;AAEA,MAAAA,KAAI,MAAM,2BAA2B,KAAK;AAC1C,aAAO,KAAK,WAAW,OAAO,OAAO,OAAO;AAAA,IAC9C;AAAA;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK,WAAW,MAAM;AAAA,EAC/B;AAAA,EAEA,YAAY,MAAc;AACxB,WAAO,KAAK,WAAW,YAAY,IAAI;AAAA,EACzC;AAAA,EAEA,gBAAgB,OAA8C;AAC5D,WAAO,KAAK,WAAW,gBAAgB,KAAK;AAAA,EAC9C;AAAA,EAEA,QAAc;AACZ,SAAK,WAAW,MAAM;AAAA,EACxB;AAAA,EAEM,OAAO,SAAoF;AAAA;AAC/F,aAAO,KAAK,WAAW,OAAO,OAAO;AAAA,IACvC;AAAA;AAAA,EAEM,OAAO,SAA2C,MAA+B;AAAA;AACrF,YAAM,KAAK,WAAW,OAAO,SAAS,IAAI;AAC1C,WAAK,cAAc;AACnB,MAAAA,KAAI,MAAM,8CAA8C;AAAA,IAC1D;AAAA;AACF;;;AEjEA,SAAsB,iBAAiB,OAAgD;AAAA;AACrF,UAAM,eAAe,IAAI,aAAa;AACtC,UAAM,aAAa,WAAW;AAE9B,eAAW,QAAQ,OAAO;AACxB,YAAM,aAAa,QAAQ,KAAK,MAAM,KAAK,cAAc;AAAA,IAC3D;AAEA,UAAM,eAAuC,CAAC;AAC9C,UAAM,SAAS,MAAM,aAAa,OAAO,CAAO,KAAK,SAAS;AAC5D,mBAAa,GAAG,IAAI;AAAA,IACtB,EAAC;AAED,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,MAAM;AAAA,IACR;AAAA,EACF;AAAA;","names":["import_logger","log"]}

package/dist/index.mjs

@@ -0,0 +1,263 @@
+var __async = (__this, __arguments, generator) => {
+  return new Promise((resolve, reject) => {
+    var fulfilled = (value) => {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var rejected = (value) => {
+      try {
+        step(generator.throw(value));
+      } catch (e) {
+        reject(e);
+      }
+    };
+    var step = (x) => x.done ? resolve(x.value) : Promise.resolve(x.value).then(fulfilled, rejected);
+    step((generator = generator.apply(__this, __arguments)).next());
+  });
+};
+
+// src/searchEngine.ts
+import { loggers as loggers2 } from "@peam-ai/logger";
+
+// src/textSearch.ts
+import { loggers } from "@peam-ai/logger";
+import { Charset, Document } from "flexsearch";
+var PEAM_DOCUMENT_IDS_KEY = "peam.documentIds";
+var MAX_DOCUMENTS_RETRIEVE = 25;
+var log = loggers.search;
+var TextSearch = class {
+  constructor() {
+    this.initialized = false;
+    this.index = this.getIndex();
+    this.documentIds = /* @__PURE__ */ new Set();
+  }
+  getIndex() {
+    return new Document({
+      worker: false,
+      document: {
+        id: "path",
+        index: ["content:title", "content:description", "content:textContent", "content:author", "content:keywords"],
+        store: true
+      },
+      tokenize: "forward",
+      resolution: 9,
+      context: {
+        resolution: 3,
+        depth: 2,
+        bidirectional: true
+      },
+      cache: 100,
+      encoder: Charset.LatinExtra
+    });
+  }
+  initialize() {
+    return __async(this, null, function* () {
+      if (this.initialized) {
+        log.debug("Text search already initialized");
+        return;
+      }
+      this.initialized = true;
+    });
+  }
+  addDocument(document) {
+    return __async(this, null, function* () {
+      if (!this.initialized) {
+        throw new Error("TextSearch not initialized. Call initialize() first.");
+      }
+      log.debug("Adding document to text search:", document.path);
+      this.index.add(document);
+      this.documentIds.add(document.path);
+    });
+  }
+  search(_0) {
+    return __async(this, arguments, function* (query, options = {}) {
+      if (!this.initialized) {
+        throw new Error("TextSearch not initialized. Call initialize() first.");
+      }
+      const limit = options.limit || MAX_DOCUMENTS_RETRIEVE;
+      const offset = options.offset || 0;
+      log.debug("Searching for:", query);
+      const results = yield this.index.search(query, {
+        limit: limit + offset,
+        suggest: options.suggest,
+        enrich: true
+      });
+      const pathSet = /* @__PURE__ */ new Set();
+      const documents = [];
+      for (const fieldResults of results) {
+        if (Array.isArray(fieldResults.result)) {
+          for (const result of fieldResults.result) {
+            const id = typeof result === "object" && "id" in result ? result.id : result;
+            const doc = typeof result === "object" && "doc" in result ? result.doc : null;
+            if (!pathSet.has(id) && doc) {
+              pathSet.add(id);
+              documents.push(doc);
+            }
+          }
+        }
+      }
+      const pagedResults = documents.slice(offset, offset + limit);
+      return pagedResults;
+    });
+  }
+  count() {
+    return this.documentIds.size;
+  }
+  getDocument(path) {
+    return this.index.get(path);
+  }
+  getAllDocuments(limit) {
+    const documents = [];
+    let count = 0;
+    limit = limit || MAX_DOCUMENTS_RETRIEVE;
+    for (const id of this.documentIds) {
+      if (count >= limit) {
+        break;
+      }
+      const doc = this.index.get(id);
+      if (doc) {
+        documents.push(doc);
+        count++;
+      }
+    }
+    log.debug("Retrieved documents from store (limit):", documents.length, limit);
+    return documents;
+  }
+  clear() {
+    this.index.clear();
+    this.index = this.getIndex();
+    this.documentIds.clear();
+  }
+  export(handler) {
+    return __async(this, null, function* () {
+      const keys = [];
+      yield handler(PEAM_DOCUMENT_IDS_KEY, JSON.stringify(Array.from(this.documentIds)));
+      keys.push(PEAM_DOCUMENT_IDS_KEY);
+      yield this.index.export((key, data) => __async(this, null, function* () {
+        keys.push(key);
+        yield handler(key, data);
+      }));
+      log.debug("Exported keys:", keys.length);
+      return { keys };
+    });
+  }
+  import(handler, keys) {
+    return __async(this, null, function* () {
+      const documentIdsData = yield handler(PEAM_DOCUMENT_IDS_KEY);
+      if (documentIdsData) {
+        const parsed = typeof documentIdsData === "string" ? JSON.parse(documentIdsData) : documentIdsData;
+        this.documentIds = new Set(parsed);
+      }
+      for (const key of keys) {
+        if (key === PEAM_DOCUMENT_IDS_KEY) {
+          continue;
+        }
+        try {
+          const data = yield handler(key);
+          if (data) {
+            this.index.import(key, data);
+          }
+        } catch (error) {
+          log.error("Error importing key:", key, error);
+        }
+      }
+      this.initialized = true;
+      log.debug("Import completed with keys:", keys.length);
+    });
+  }
+};
+
+// src/searchEngine.ts
+var log2 = loggers2.search;
+var SearchEngine = class {
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  constructor(_config) {
+    this.textSearch = new TextSearch();
+    this.initialized = false;
+  }
+  initialize() {
+    return __async(this, null, function* () {
+      if (this.initialized) {
+        return;
+      }
+      log2.debug("Initializing search engine");
+      yield this.textSearch.initialize();
+      this.initialized = true;
+    });
+  }
+  addPage(path, content) {
+    return __async(this, null, function* () {
+      if (!this.initialized) {
+        throw new Error("Search engine not initialized. Call initialize() first.");
+      }
+      const document = {
+        id: path,
+        path,
+        content
+      };
+      yield this.textSearch.addDocument(document);
+      log2.debug("Page added to search engine:", path);
+    });
+  }
+  search(_0) {
+    return __async(this, arguments, function* (query, options = {}) {
+      if (!this.initialized) {
+        throw new Error("Search engine not initialized. Call initialize() first.");
+      }
+      log2.debug("Performing text search:", query);
+      return this.textSearch.search(query, options);
+    });
+  }
+  count() {
+    return this.textSearch.count();
+  }
+  getDocument(path) {
+    return this.textSearch.getDocument(path);
+  }
+  getAllDocuments(limit) {
+    return this.textSearch.getAllDocuments(limit);
+  }
+  clear() {
+    this.textSearch.clear();
+  }
+  export(handler) {
+    return __async(this, null, function* () {
+      return this.textSearch.export(handler);
+    });
+  }
+  import(handler, keys) {
+    return __async(this, null, function* () {
+      yield this.textSearch.import(handler, keys);
+      this.initialized = true;
+      log2.debug("Search engine initialized from imported data");
+    });
+  }
+};
+
+// src/indexBuilder.ts
+function buildSearchIndex(pages) {
+  return __async(this, null, function* () {
+    const searchEngine = new SearchEngine();
+    yield searchEngine.initialize();
+    for (const page of pages) {
+      yield searchEngine.addPage(page.path, page.structuredPage);
+    }
+    const exportedData = {};
+    const result = yield searchEngine.export((key, data) => __async(null, null, function* () {
+      exportedData[key] = data;
+    }));
+    return {
+      keys: result.keys,
+      data: exportedData
+    };
+  });
+}
+export {
+  SearchEngine,
+  TextSearch,
+  buildSearchIndex
+};
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/searchEngine.ts","../src/textSearch.ts","../src/indexBuilder.ts"],"sourcesContent":["import { loggers } from '@peam-ai/logger';\nimport type { StructuredPage } from '@peam-ai/parser';\nimport { TextSearch, type TextSearchOptions } from './textSearch';\nimport type { StructuredPageDocumentData } from './types';\n\nconst log = loggers.search;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-object-type\nexport interface SearchEngineConfig {\n  // Reserved for future configuration options\n}\n\nexport class SearchEngine {\n  private textSearch: TextSearch;\n  private initialized: boolean;\n\n  // eslint-disable-next-line @typescript-eslint/no-unused-vars\n  constructor(_config?: SearchEngineConfig) {\n    this.textSearch = new TextSearch();\n    this.initialized = false;\n  }\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      return;\n    }\n\n    log.debug('Initializing search engine');\n    await this.textSearch.initialize();\n    this.initialized = true;\n  }\n\n  async addPage(path: string, content: StructuredPage): Promise<void> {\n    if (!this.initialized) {\n      throw new Error('Search engine not initialized. Call initialize() first.');\n    }\n\n    const document: StructuredPageDocumentData = {\n      id: path,\n      path,\n      content,\n    };\n\n    await this.textSearch.addDocument(document);\n    log.debug('Page added to search engine:', path);\n  }\n\n  async search(query: string, options: TextSearchOptions = {}): Promise<StructuredPageDocumentData[]> {\n    if (!this.initialized) {\n      throw new Error('Search engine not initialized. Call initialize() first.');\n    }\n\n    log.debug('Performing text search:', query);\n    return this.textSearch.search(query, options);\n  }\n\n  count(): number {\n    return this.textSearch.count();\n  }\n\n  getDocument(path: string) {\n    return this.textSearch.getDocument(path);\n  }\n\n  getAllDocuments(limit?: number): StructuredPageDocumentData[] {\n    return this.textSearch.getAllDocuments(limit);\n  }\n\n  clear(): void {\n    this.textSearch.clear();\n  }\n\n  async export(handler: (key: string, data: string) => Promise<void>): Promise<{ keys: string[] }> {\n    return this.textSearch.export(handler);\n  }\n\n  async import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void> {\n    await this.textSearch.import(handler, keys);\n    this.initialized = true;\n    log.debug('Search engine initialized from imported data');\n  }\n}\n","import { loggers } from '@peam-ai/logger';\nimport { Charset, Document } from 'flexsearch';\nimport type { StructuredPageDocumentData } from './types';\n\nexport interface TextSearchOptions {\n  limit?: number;\n  offset?: number;\n  suggest?: boolean;\n}\n\nconst PEAM_DOCUMENT_IDS_KEY = 'peam.documentIds';\nconst MAX_DOCUMENTS_RETRIEVE = 25;\nconst log = loggers.search;\n\nexport class TextSearch {\n  private index: Document<StructuredPageDocumentData>;\n  private initialized: boolean;\n  private documentIds: Set<string>;\n\n  constructor() {\n    this.initialized = false;\n    this.index = this.getIndex();\n    this.documentIds = new Set();\n  }\n\n  private getIndex() {\n    return new Document<StructuredPageDocumentData>({\n      worker: false,\n      document: {\n        id: 'path',\n        index: ['content:title', 'content:description', 'content:textContent', 'content:author', 'content:keywords'],\n        store: true,\n      },\n      tokenize: 'forward',\n      resolution: 9,\n      context: {\n        resolution: 3,\n        depth: 2,\n        bidirectional: true,\n      },\n      cache: 100,\n      encoder: Charset.LatinExtra,\n    });\n  }\n\n  async initialize(): Promise<void> {\n    if (this.initialized) {\n      log.debug('Text search already initialized');\n      return;\n    }\n\n    this.initialized = true;\n  }\n\n  async addDocument(document: StructuredPageDocumentData): Promise<void> {\n    if (!this.initialized) {\n      throw new Error('TextSearch not initialized. Call initialize() first.');\n    }\n\n    log.debug('Adding document to text search:', document.path);\n\n    this.index.add(document);\n    this.documentIds.add(document.path);\n  }\n\n  async search(query: string, options: TextSearchOptions = {}): Promise<StructuredPageDocumentData[]> {\n    if (!this.initialized) {\n      throw new Error('TextSearch not initialized. Call initialize() first.');\n    }\n\n    const limit = options.limit || MAX_DOCUMENTS_RETRIEVE;\n    const offset = options.offset || 0;\n\n    log.debug('Searching for:', query);\n\n    const results = await this.index.search(query, {\n      limit: limit + offset,\n      suggest: options.suggest,\n      enrich: true,\n    });\n\n    const pathSet = new Set<string>();\n    const documents: StructuredPageDocumentData[] = [];\n\n    for (const fieldResults of results) {\n      if (Array.isArray(fieldResults.result)) {\n        for (const result of fieldResults.result) {\n          const id = typeof result === 'object' && 'id' in result ? result.id : result;\n          const doc = typeof result === 'object' && 'doc' in result ? result.doc : null;\n\n          if (!pathSet.has(id as string) && doc) {\n            pathSet.add(id as string);\n            documents.push(doc);\n          }\n        }\n      }\n    }\n\n    const pagedResults = documents.slice(offset, offset + limit);\n\n    return pagedResults;\n  }\n\n  count(): number {\n    return this.documentIds.size;\n  }\n\n  getDocument(path: string) {\n    return this.index.get(path);\n  }\n\n  getAllDocuments(limit?: number): StructuredPageDocumentData[] {\n    const documents: StructuredPageDocumentData[] = [];\n    let count = 0;\n    limit = limit || MAX_DOCUMENTS_RETRIEVE;\n\n    for (const id of this.documentIds) {\n      if (count >= limit) {\n        break;\n      }\n\n      const doc = this.index.get(id);\n      if (doc) {\n        documents.push(doc);\n        count++;\n      }\n    }\n\n    log.debug('Retrieved documents from store (limit):', documents.length, limit);\n    return documents;\n  }\n\n  clear(): void {\n    this.index.clear();\n    this.index = this.getIndex();\n    this.documentIds.clear();\n  }\n\n  async export(handler: (key: string, data: string) => Promise<void>): Promise<{ keys: string[] }> {\n    const keys: string[] = [];\n\n    await handler(PEAM_DOCUMENT_IDS_KEY, JSON.stringify(Array.from(this.documentIds)));\n    keys.push(PEAM_DOCUMENT_IDS_KEY);\n\n    await this.index.export(async (key: string, data: string) => {\n      keys.push(key);\n      await handler(key, data);\n    });\n\n    log.debug('Exported keys:', keys.length);\n\n    return { keys };\n  }\n\n  async import(handler: (key: string) => Promise<string>, keys: string[]): Promise<void> {\n    const documentIdsData = await handler(PEAM_DOCUMENT_IDS_KEY);\n    if (documentIdsData) {\n      const parsed = typeof documentIdsData === 'string' ? JSON.parse(documentIdsData) : documentIdsData;\n      this.documentIds = new Set(parsed);\n    }\n\n    for (const key of keys) {\n      if (key === PEAM_DOCUMENT_IDS_KEY) {\n        continue;\n      }\n\n      try {\n        const data = await handler(key);\n        if (data) {\n          this.index.import(key, data);\n        }\n      } catch (error) {\n        log.error('Error importing key:', key, error);\n      }\n    }\n\n    this.initialized = true;\n    log.debug('Import completed with keys:', keys.length);\n  }\n}\n","import type { StructuredPage } from '@peam-ai/parser';\nimport { SearchEngine } from './searchEngine';\n\nexport interface PageToIndex {\n  path: string;\n  structuredPage: StructuredPage;\n}\n\nexport interface SearchIndexData {\n  keys: string[];\n  data: Record<string, string>;\n}\n\n/**\n * Build a search index from structured pages\n */\nexport async function buildSearchIndex(pages: PageToIndex[]): Promise<SearchIndexData> {\n  const searchEngine = new SearchEngine();\n  await searchEngine.initialize();\n\n  for (const page of pages) {\n    await searchEngine.addPage(page.path, page.structuredPage);\n  }\n\n  const exportedData: Record<string, string> = {};\n  const result = await searchEngine.export(async (key, data) => {\n    exportedData[key] = data;\n  });\n\n  return {\n    keys: result.keys,\n    data: exportedData,\n  };\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,SAAS,WAAAA,gBAAe;;;ACAxB,SAAS,eAAe;AACxB,SAAS,SAAS,gBAAgB;AASlC,IAAM,wBAAwB;AAC9B,IAAM,yBAAyB;AAC/B,IAAM,MAAM,QAAQ;AAEb,IAAM,aAAN,MAAiB;AAAA,EAKtB,cAAc;AACZ,SAAK,cAAc;AACnB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,cAAc,oBAAI,IAAI;AAAA,EAC7B;AAAA,EAEQ,WAAW;AACjB,WAAO,IAAI,SAAqC;AAAA,MAC9C,QAAQ;AAAA,MACR,UAAU;AAAA,QACR,IAAI;AAAA,QACJ,OAAO,CAAC,iBAAiB,uBAAuB,uBAAuB,kBAAkB,kBAAkB;AAAA,QAC3G,OAAO;AAAA,MACT;AAAA,MACA,UAAU;AAAA,MACV,YAAY;AAAA,MACZ,SAAS;AAAA,QACP,YAAY;AAAA,QACZ,OAAO;AAAA,QACP,eAAe;AAAA,MACjB;AAAA,MACA,OAAO;AAAA,MACP,SAAS,QAAQ;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEM,aAA4B;AAAA;AAChC,UAAI,KAAK,aAAa;AACpB,YAAI,MAAM,iCAAiC;AAC3C;AAAA,MACF;AAEA,WAAK,cAAc;AAAA,IACrB;AAAA;AAAA,EAEM,YAAY,UAAqD;AAAA;AACrE,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,sDAAsD;AAAA,MACxE;AAEA,UAAI,MAAM,mCAAmC,SAAS,IAAI;AAE1D,WAAK,MAAM,IAAI,QAAQ;AACvB,WAAK,YAAY,IAAI,SAAS,IAAI;AAAA,IACpC;AAAA;AAAA,EAEM,OAAO,IAAuF;AAAA,+CAAvF,OAAe,UAA6B,CAAC,GAA0C;AAClG,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,sDAAsD;AAAA,MACxE;AAEA,YAAM,QAAQ,QAAQ,SAAS;AAC/B,YAAM,SAAS,QAAQ,UAAU;AAEjC,UAAI,MAAM,kBAAkB,KAAK;AAEjC,YAAM,UAAU,MAAM,KAAK,MAAM,OAAO,OAAO;AAAA,QAC7C,OAAO,QAAQ;AAAA,QACf,SAAS,QAAQ;AAAA,QACjB,QAAQ;AAAA,MACV,CAAC;AAED,YAAM,UAAU,oBAAI,IAAY;AAChC,YAAM,YAA0C,CAAC;AAEjD,iBAAW,gBAAgB,SAAS;AAClC,YAAI,MAAM,QAAQ,aAAa,MAAM,GAAG;AACtC,qBAAW,UAAU,aAAa,QAAQ;AACxC,kBAAM,KAAK,OAAO,WAAW,YAAY,QAAQ,SAAS,OAAO,KAAK;AACtE,kBAAM,MAAM,OAAO,WAAW,YAAY,SAAS,SAAS,OAAO,MAAM;AAEzE,gBAAI,CAAC,QAAQ,IAAI,EAAY,KAAK,KAAK;AACrC,sBAAQ,IAAI,EAAY;AACxB,wBAAU,KAAK,GAAG;AAAA,YACpB;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,YAAM,eAAe,UAAU,MAAM,QAAQ,SAAS,KAAK;AAE3D,aAAO;AAAA,IACT;AAAA;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK,YAAY;AAAA,EAC1B;AAAA,EAEA,YAAY,MAAc;AACxB,WAAO,KAAK,MAAM,IAAI,IAAI;AAAA,EAC5B;AAAA,EAEA,gBAAgB,OAA8C;AAC5D,UAAM,YAA0C,CAAC;AACjD,QAAI,QAAQ;AACZ,YAAQ,SAAS;AAEjB,eAAW,MAAM,KAAK,aAAa;AACjC,UAAI,SAAS,OAAO;AAClB;AAAA,MACF;AAEA,YAAM,MAAM,KAAK,MAAM,IAAI,EAAE;AAC7B,UAAI,KAAK;AACP,kBAAU,KAAK,GAAG;AAClB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,MAAM,2CAA2C,UAAU,QAAQ,KAAK;AAC5E,WAAO;AAAA,EACT;AAAA,EAEA,QAAc;AACZ,SAAK,MAAM,MAAM;AACjB,SAAK,QAAQ,KAAK,SAAS;AAC3B,SAAK,YAAY,MAAM;AAAA,EACzB;AAAA,EAEM,OAAO,SAAoF;AAAA;AAC/F,YAAM,OAAiB,CAAC;AAExB,YAAM,QAAQ,uBAAuB,KAAK,UAAU,MAAM,KAAK,KAAK,WAAW,CAAC,CAAC;AACjF,WAAK,KAAK,qBAAqB;AAE/B,YAAM,KAAK,MAAM,OAAO,CAAO,KAAa,SAAiB;AAC3D,aAAK,KAAK,GAAG;AACb,cAAM,QAAQ,KAAK,IAAI;AAAA,MACzB,EAAC;AAED,UAAI,MAAM,kBAAkB,KAAK,MAAM;AAEvC,aAAO,EAAE,KAAK;AAAA,IAChB;AAAA;AAAA,EAEM,OAAO,SAA2C,MAA+B;AAAA;AACrF,YAAM,kBAAkB,MAAM,QAAQ,qBAAqB;AAC3D,UAAI,iBAAiB;AACnB,cAAM,SAAS,OAAO,oBAAoB,WAAW,KAAK,MAAM,eAAe,IAAI;AACnF,aAAK,cAAc,IAAI,IAAI,MAAM;AAAA,MACnC;AAEA,iBAAW,OAAO,MAAM;AACtB,YAAI,QAAQ,uBAAuB;AACjC;AAAA,QACF;AAEA,YAAI;AACF,gBAAM,OAAO,MAAM,QAAQ,GAAG;AAC9B,cAAI,MAAM;AACR,iBAAK,MAAM,OAAO,KAAK,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,OAAO;AACd,cAAI,MAAM,wBAAwB,KAAK,KAAK;AAAA,QAC9C;AAAA,MACF;AAEA,WAAK,cAAc;AACnB,UAAI,MAAM,+BAA+B,KAAK,MAAM;AAAA,IACtD;AAAA;AACF;;;AD9KA,IAAMC,OAAMC,SAAQ;AAOb,IAAM,eAAN,MAAmB;AAAA;AAAA,EAKxB,YAAY,SAA8B;AACxC,SAAK,aAAa,IAAI,WAAW;AACjC,SAAK,cAAc;AAAA,EACrB;AAAA,EAEM,aAA4B;AAAA;AAChC,UAAI,KAAK,aAAa;AACpB;AAAA,MACF;AAEA,MAAAD,KAAI,MAAM,4BAA4B;AACtC,YAAM,KAAK,WAAW,WAAW;AACjC,WAAK,cAAc;AAAA,IACrB;AAAA;AAAA,EAEM,QAAQ,MAAc,SAAwC;AAAA;AAClE,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,yDAAyD;AAAA,MAC3E;AAEA,YAAM,WAAuC;AAAA,QAC3C,IAAI;AAAA,QACJ;AAAA,QACA;AAAA,MACF;AAEA,YAAM,KAAK,WAAW,YAAY,QAAQ;AAC1C,MAAAA,KAAI,MAAM,gCAAgC,IAAI;AAAA,IAChD;AAAA;AAAA,EAEM,OAAO,IAAuF;AAAA,+CAAvF,OAAe,UAA6B,CAAC,GAA0C;AAClG,UAAI,CAAC,KAAK,aAAa;AACrB,cAAM,IAAI,MAAM,yDAAyD;AAAA,MAC3E;AAEA,MAAAA,KAAI,MAAM,2BAA2B,KAAK;AAC1C,aAAO,KAAK,WAAW,OAAO,OAAO,OAAO;AAAA,IAC9C;AAAA;AAAA,EAEA,QAAgB;AACd,WAAO,KAAK,WAAW,MAAM;AAAA,EAC/B;AAAA,EAEA,YAAY,MAAc;AACxB,WAAO,KAAK,WAAW,YAAY,IAAI;AAAA,EACzC;AAAA,EAEA,gBAAgB,OAA8C;AAC5D,WAAO,KAAK,WAAW,gBAAgB,KAAK;AAAA,EAC9C;AAAA,EAEA,QAAc;AACZ,SAAK,WAAW,MAAM;AAAA,EACxB;AAAA,EAEM,OAAO,SAAoF;AAAA;AAC/F,aAAO,KAAK,WAAW,OAAO,OAAO;AAAA,IACvC;AAAA;AAAA,EAEM,OAAO,SAA2C,MAA+B;AAAA;AACrF,YAAM,KAAK,WAAW,OAAO,SAAS,IAAI;AAC1C,WAAK,cAAc;AACnB,MAAAA,KAAI,MAAM,8CAA8C;AAAA,IAC1D;AAAA;AACF;;;AEjEA,SAAsB,iBAAiB,OAAgD;AAAA;AACrF,UAAM,eAAe,IAAI,aAAa;AACtC,UAAM,aAAa,WAAW;AAE9B,eAAW,QAAQ,OAAO;AACxB,YAAM,aAAa,QAAQ,KAAK,MAAM,KAAK,cAAc;AAAA,IAC3D;AAEA,UAAM,eAAuC,CAAC;AAC9C,UAAM,SAAS,MAAM,aAAa,OAAO,CAAO,KAAK,SAAS;AAC5D,mBAAa,GAAG,IAAI;AAAA,IACtB,EAAC;AAED,WAAO;AAAA,MACL,MAAM,OAAO;AAAA,MACb,MAAM;AAAA,IACR;AAAA,EACF;AAAA;","names":["loggers","log","loggers"]}

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@peam-ai/search",
+  "version": "0.1.1",
+  "description": "Vector database and search functionality for Peam",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "types": "./dist/index.d.ts",
+  "homepage": "https://peam.ai",
+  "author": "Afshin Mehrabani",
+  "license": "Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/peam-ai/peam.git",
+    "directory": "packages/search"
+  },
+  "bugs": {
+    "url": "https://github.com/peam-ai/peam/issues"
+  },
+  "files": [
+    "dist/**/*"
+  ],
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "flexsearch": "^0.8.212",
+    "@peam-ai/logger": "0.1.1",
+    "@peam-ai/parser": "0.1.1"
+  },
+  "devDependencies": {
+    "@types/node": "^22.10.2",
+    "tsup": "^8.2.4",
+    "typescript": "^5.5.4"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "clean": "rm -rf dist",
+    "lint": "eslint .",
+    "test:eslint": "eslint \"src/**/*.ts*\"",
+    "test:prettier": "prettier --check \"src/**/*.ts*\"",
+    "test:jest": "jest"
+  }
+}