@cvfile/embed 0.1.0

package/LICENSE

@@ -0,0 +1,190 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for describing the origin of the Work and
+      reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Support. While redistributing the Work or
+      Derivative Works thereof, You may choose to offer, and charge a
+      fee for, acceptance of support, warranty, indemnity, or other
+      liability obligations and/or rights consistent with this License.
+      However, in accepting such obligations, You may act only on Your
+      own behalf and on Your sole responsibility, not on behalf of any
+      other Contributor, and only if You agree to indemnify, defend,
+      and hold each Contributor harmless for any liability incurred by,
+      or claims asserted against, such Contributor by reason of your
+      accepting any such warranty or support.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2026 The cvfile.org authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

package/dist/index.cjs

@@ -0,0 +1,303 @@
+'use strict';
+
+var transformers = require('@huggingface/transformers');
+
+// src/chunk.ts
+var HEADING = /^(#{1,6})\s+(.+?)\s*$/;
+function chunkMarkdown(markdown, opts = {}) {
+  const mode = opts.mode ?? "section";
+  if (mode === "document") {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  if (mode === "paragraph") {
+    return paragraphChunks(markdown);
+  }
+  return sectionChunks(markdown);
+}
+function sectionChunks(markdown) {
+  const lines = splitWithOffsets(markdown);
+  const sections = [];
+  let current = null;
+  const ids = /* @__PURE__ */ new Set();
+  function flush(end) {
+    if (!current) return;
+    const text = markdown.slice(current.start, end);
+    if (text.trim().length === 0) {
+      current = null;
+      return;
+    }
+    sections.push({ id: current.id, textOffset: current.start, textLength: end - current.start, text });
+    current = null;
+  }
+  for (const line of lines) {
+    const match = HEADING.exec(line.text);
+    if (match) {
+      flush(line.offset);
+      const id = uniqueId(slugify(match[2] ?? `section-${sections.length + 1}`), ids);
+      ids.add(id);
+      current = { id, start: line.offset, end: line.offset + line.text.length };
+      continue;
+    }
+    if (current === null) {
+      const id = uniqueId("preamble", ids);
+      ids.add(id);
+      current = { id, start: line.offset, end: line.offset + line.text.length };
+    } else {
+      current.end = line.offset + line.text.length;
+    }
+  }
+  flush(markdown.length);
+  if (sections.length === 0) {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  return sections;
+}
+function paragraphChunks(markdown) {
+  const out = [];
+  const ids = /* @__PURE__ */ new Set();
+  let cursor = 0;
+  let i = 0;
+  while (cursor < markdown.length) {
+    let end = markdown.indexOf("\n\n", cursor);
+    if (end === -1) end = markdown.length;
+    const text = markdown.slice(cursor, end);
+    if (text.trim().length > 0) {
+      const id = uniqueId(slugify(text.split("\n")[0] ?? `p-${i}`), ids);
+      ids.add(id);
+      out.push({ id, textOffset: cursor, textLength: text.length, text });
+      i += 1;
+    }
+    cursor = end + 2;
+  }
+  if (out.length === 0) {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  return out;
+}
+function splitWithOffsets(s) {
+  const lines = [];
+  let offset = 0;
+  for (const line of s.split("\n")) {
+    const withNl = line + (offset + line.length < s.length ? "\n" : "");
+    lines.push({ text: withNl, offset });
+    offset += withNl.length;
+  }
+  return lines;
+}
+function slugify(s) {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || "section";
+}
+function uniqueId(base, taken) {
+  if (!taken.has(base)) return base;
+  let n = 2;
+  while (taken.has(`${base}-${n}`)) n += 1;
+  return `${base}-${n}`;
+}
+function createTransformersBackend(opts) {
+  if (opts.allowRemoteModels === false) {
+    transformers.env.allowRemoteModels = false;
+  }
+  let pipelinePromise = null;
+  let resolvedDimension = null;
+  function getPipeline() {
+    if (!pipelinePromise) {
+      const modelOpts = { dtype: opts.dtype ?? "fp32" };
+      if (opts.device !== void 0) modelOpts.device = opts.device;
+      pipelinePromise = transformers.pipeline("feature-extraction", opts.model, modelOpts);
+    }
+    return pipelinePromise;
+  }
+  const backend = {
+    model: opts.model,
+    modelRevision: opts.modelRevision ?? "main",
+    metric: opts.metric ?? "cosine",
+    normalized: true,
+    async embed(texts) {
+      if (texts.length === 0) {
+        return { vectors: [], dimension: opts.dimension ?? 0 };
+      }
+      const pipe = await getPipeline();
+      const tensor = await pipe(texts, { pooling: "mean", normalize: true });
+      const data = tensor.data;
+      const dimension = tensor.dims[tensor.dims.length - 1];
+      if (resolvedDimension !== null && resolvedDimension !== dimension) {
+        throw new Error(`Model emitted inconsistent dimension: had ${resolvedDimension}, now ${dimension}`);
+      }
+      resolvedDimension = dimension;
+      const vectors = [];
+      for (let i = 0; i < texts.length; i += 1) {
+        vectors.push(new Float32Array(data.buffer, data.byteOffset + i * dimension * 4, dimension).slice());
+      }
+      return { vectors, dimension };
+    }
+  };
+  if (opts.dimension !== void 0) backend.dimension = opts.dimension;
+  return backend;
+}
+
+// src/types.ts
+var DEFAULT_MODEL = "Xenova/bge-m3";
+var DEFAULT_MODEL_DIMENSION = 1024;
+
+// src/embed.ts
+async function embed(markdown, opts = {}) {
+  const chunks = chunkMarkdown(markdown, { mode: opts.chunking ?? "section" });
+  const backend = opts.backend ?? createTransformersBackend(
+    opts.modelRevision !== void 0 ? { model: opts.model ?? DEFAULT_MODEL, modelRevision: opts.modelRevision } : { model: opts.model ?? DEFAULT_MODEL }
+  );
+  const matrix = await backend.embed(chunks.map((c) => c.text));
+  if (matrix.vectors.length !== chunks.length) {
+    throw new Error(`Backend returned ${matrix.vectors.length} vectors for ${chunks.length} chunks`);
+  }
+  const embeddingChunks = chunks.map((c, i) => ({
+    id: c.id,
+    textOffset: c.textOffset,
+    textLength: c.textLength,
+    vector: matrix.vectors[i]
+  }));
+  const space = {
+    model: backend.model,
+    modelRevision: backend.modelRevision,
+    dimension: matrix.dimension,
+    metric: backend.metric,
+    normalized: backend.normalized,
+    chunking: opts.chunking ?? "section",
+    chunks: embeddingChunks
+  };
+  return { formatVersion: 1, spaces: [space] };
+}
+
+// src/search.ts
+function searchSemantic(payload, queryVector, opts = {}) {
+  const space = pickSpace(payload, opts.model);
+  if (!space) throw new Error("No matching embedding space found");
+  if (queryVector.length !== space.dimension) {
+    throw new Error(`Query vector dimension ${queryVector.length} does not match space ${space.model} (${space.dimension})`);
+  }
+  const k = opts.k ?? 5;
+  const scored = space.chunks.map((c) => ({
+    spaceModel: space.model,
+    chunkId: c.id,
+    textOffset: c.textOffset,
+    textLength: c.textLength,
+    score: similarity(queryVector, c.vector, space.metric)
+  }));
+  const order = space.metric === "euclidean" ? 1 : -1;
+  scored.sort((a, b) => order * (a.score - b.score));
+  return scored.slice(0, k);
+}
+function pickSpace(payload, modelHint) {
+  if (modelHint) return payload.spaces.find((s) => s.model === modelHint);
+  return payload.spaces[0];
+}
+function similarity(a, b, metric) {
+  if (metric === "euclidean") {
+    let sum = 0;
+    for (let i = 0; i < a.length; i += 1) {
+      const d = a[i] - b[i];
+      sum += d * d;
+    }
+    return Math.sqrt(sum);
+  }
+  let dot = 0;
+  for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
+  if (metric === "dot") return dot;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+// src/huggingface-backend.ts
+var DEFAULT_BASE_URL = "https://router.huggingface.co/hf-inference/models";
+function createHuggingFaceBackend(opts) {
+  const token = opts.token ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_TOKEN;
+  if (!token) {
+    throw new Error("HF_TOKEN (or HUGGINGFACE_TOKEN) is required for the Hugging Face backend");
+  }
+  const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
+  const url = `${baseUrl}/${encodeURI(opts.model)}/pipeline/feature-extraction`;
+  const backend = {
+    model: opts.model,
+    modelRevision: opts.modelRevision ?? "main",
+    metric: opts.metric ?? "cosine",
+    normalized: true,
+    async embed(texts) {
+      if (texts.length === 0) {
+        return { vectors: [], dimension: opts.dimension ?? 0 };
+      }
+      const body = JSON.stringify({ inputs: texts, options: { wait_for_model: true } });
+      const res = await fetch(url, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${token}`
+        },
+        body
+      });
+      if (!res.ok) {
+        const detail = await res.text().catch(() => "");
+        throw new Error(`HF Inference API ${res.status} for ${opts.model}: ${detail || res.statusText}`);
+      }
+      const raw = await res.json();
+      const matrix = parseHfResponse(raw, texts.length);
+      const dimension = matrix[0]?.length ?? opts.dimension ?? 0;
+      const vectors = matrix.map((v) => normalize(Float32Array.from(v)));
+      return { vectors, dimension };
+    }
+  };
+  if (opts.dimension !== void 0) backend.dimension = opts.dimension;
+  return backend;
+}
+function parseHfResponse(raw, expectedCount) {
+  if (!Array.isArray(raw)) {
+    throw new Error("HF Inference API: expected array response");
+  }
+  if (raw.length === 0) return [];
+  const first = raw[0];
+  if (typeof first === "number") {
+    if (expectedCount !== 1) {
+      throw new Error(`HF Inference API: got 1 vector, expected ${expectedCount}`);
+    }
+    return [raw];
+  }
+  if (Array.isArray(first) && (first.length === 0 || typeof first[0] === "number")) {
+    return raw;
+  }
+  if (Array.isArray(first) && Array.isArray(first[0])) {
+    return raw.map(meanPool);
+  }
+  throw new Error("HF Inference API: unrecognised response shape");
+}
+function meanPool(tokens) {
+  if (tokens.length === 0) return [];
+  const dim = tokens[0].length;
+  const out = new Array(dim).fill(0);
+  for (const t of tokens) {
+    for (let i = 0; i < dim; i += 1) out[i] += t[i];
+  }
+  for (let i = 0; i < dim; i += 1) out[i] /= tokens.length;
+  return out;
+}
+function normalize(v) {
+  let sum = 0;
+  for (let i = 0; i < v.length; i += 1) sum += v[i] * v[i];
+  const norm = Math.sqrt(sum);
+  if (norm === 0) return v;
+  for (let i = 0; i < v.length; i += 1) v[i] = v[i] / norm;
+  return v;
+}
+
+exports.DEFAULT_MODEL = DEFAULT_MODEL;
+exports.DEFAULT_MODEL_DIMENSION = DEFAULT_MODEL_DIMENSION;
+exports.chunkMarkdown = chunkMarkdown;
+exports.createHuggingFaceBackend = createHuggingFaceBackend;
+exports.createTransformersBackend = createTransformersBackend;
+exports.embed = embed;
+exports.searchSemantic = searchSemantic;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/chunk.ts","../src/transformers-backend.ts","../src/types.ts","../src/embed.ts","../src/search.ts","../src/huggingface-backend.ts"],"names":["env","pipeline"],"mappings":";;;;;AAsBA,IAAM,OAAA,GAAU,uBAAA;AAET,SAAS,aAAA,CAAc,QAAA,EAAkB,IAAA,GAAqB,EAAC,EAAoB;AACxF,EAAA,MAAM,IAAA,GAAO,KAAK,IAAA,IAAQ,SAAA;AAC1B,EAAA,IAAI,SAAS,UAAA,EAAY;AACvB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,IAAI,SAAS,WAAA,EAAa;AACxB,IAAA,OAAO,gBAAgB,QAAQ,CAAA;AAAA,EACjC;AACA,EAAA,OAAO,cAAc,QAAQ,CAAA;AAC/B;AAEA,SAAS,cAAc,QAAA,EAAmC;AACxD,EAAA,MAAM,KAAA,GAAQ,iBAAiB,QAAQ,CAAA;AACvC,EAAA,MAAM,WAA4B,EAAC;AACnC,EAAA,IAAI,OAAA,GAA6D,IAAA;AACjE,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAE5B,EAAA,SAAS,MAAM,GAAA,EAAmB;AAChC,IAAA,IAAI,CAAC,OAAA,EAAS;AACd,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,KAAA,CAAM,OAAA,CAAQ,OAAO,GAAG,CAAA;AAC9C,IAAA,IAAI,IAAA,CAAK,IAAA,EAAK,CAAE,MAAA,KAAW,CAAA,EAAG;AAC5B,MAAA,OAAA,GAAU,IAAA;AACV,MAAA;AAAA,IACF;AACA,IAAA,QAAA,CAAS,IAAA,CAAK,EAAE,EAAA,EAAI,OAAA,CAAQ,EAAA,EAAI,UAAA,EAAY,OAAA,CAAQ,KAAA,EAAO,UAAA,EAAY,GAAA,GAAM,OAAA,CAAQ,KAAA,EAAO,MAAM,CAAA;AAClG,IAAA,OAAA,GAAU,IAAA;AAAA,EACZ;AAEA,EAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,IAAA,CAAK,IAAA,CAAK,IAAI,CAAA;AACpC,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,KAAA,CAAM,KAAK,MAAM,CAAA;AACjB,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,OAAA,CAAQ,KAAA,CAAM,CAAC,CAAA,IAAK,CAAA,QAAA,EAAW,QAAA,CAAS,MAAA,GAAS,CAAC,CAAA,CAAE,CAAA,EAAG,GAAG,CAAA;AAC9E,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,OAAA,GAAU,EAAE,EAAA,EAAI,KAAA,EAAO,IAAA,CAAK,MAAA,EAAQ,KAAK,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA,EAAO;AACxE,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAY,IAAA,EAAM;AACpB,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,UAAA,EAAY,GAAG,CAAA;AACnC,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,OAAA,GAAU,EAAE,EAAA,EAAI,KAAA,EAAO,IAAA,CAAK,MAAA,EAAQ,KAAK,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA,EAAO;AAAA,IAC1E,CAAA,MAAO;AACL,MAAA,OAAA,CAAQ,GAAA,GAAM,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA;AAAA,IACxC;AAAA,EACF;AACA,EAAA,KAAA,CAAM,SAAS,MAAM,CAAA;AAErB,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,QAAA;AACT;AAEA,SAAS,gBAAgB,QAAA,EAAmC;AAC1D,EAAA,MAAM,MAAuB,EAAC;AAC9B,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAC5B,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,OAAO,MAAA,GAAS,SAAS,MAAA,EAAQ;AAC/B,IAAA,IAAI,GAAA,GAAM,QAAA,CAAS,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AACzC,IAAA,IAAI,GAAA,KAAQ,EAAA,EAAI,GAAA,GAAM,QAAA,CAAS,MAAA;AAC/B,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,KAAA,CAAM,MAAA,EAAQ,GAAG,CAAA;AACvC,IAAA,IAAI,IAAA,CAAK,IAAA,EAAK,CAAE,MAAA,GAAS,CAAA,EAAG;AAC1B,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA,CAAE,CAAC,CAAA,IAAK,CAAA,EAAA,EAAK,CAAC,CAAA,CAAE,GAAG,GAAG,CAAA;AACjE,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,GAAA,CAAI,IAAA,CAAK,EAAE,EAAA,EAAI,UAAA,EAAY,QAAQ,UAAA,EAAY,IAAA,CAAK,MAAA,EAAQ,IAAA,EAAM,CAAA;AAClE,MAAA,CAAA,IAAK,CAAA;AAAA,IACP;AACA,IAAA,MAAA,GAAS,GAAA,GAAM,CAAA;AAAA,EACjB;AACA,EAAA,IAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACpB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,GAAA;AACT;AAEA,SAAS,iBAAiB,CAAA,EAA+C;AACvE,EAAA,MAAM,QAA4C,EAAC;AACnD,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,KAAA,MAAW,IAAA,IAAQ,CAAA,CAAE,KAAA,CAAM,IAAI,CAAA,EAAG;AAChC,IAAA,MAAM,SAAS,IAAA,IAAQ,MAAA,GAAS,KAAK,MAAA,GAAS,CAAA,CAAE,SAAS,IAAA,GAAO,EAAA,CAAA;AAChE,IAAA,KAAA,CAAM,IAAA,CAAK,EAAE,IAAA,EAAM,MAAA,EAAQ,QAAQ,CAAA;AACnC,IAAA,MAAA,IAAU,MAAA,CAAO,MAAA;AAAA,EACnB;AACA,EAAA,OAAO,KAAA;AACT;AAEA,SAAS,QAAQ,CAAA,EAAmB;AAClC,EAAA,OACE,CAAA,CACG,WAAA,EAAY,CACZ,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CACtB,KAAA,CAAM,CAAA,EAAG,EAAE,CAAA,IAAK,SAAA;AAEvB;AAEA,SAAS,QAAA,CAAS,MAAc,KAAA,EAA4B;AAC1D,EAAA,IAAI,CAAC,KAAA,CAAM,GAAA,CAAI,IAAI,GAAG,OAAO,IAAA;AAC7B,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,OAAO,KAAA,CAAM,IAAI,CAAA,EAAG,IAAI,IAAI,CAAC,CAAA,CAAE,GAAG,CAAA,IAAK,CAAA;AACvC,EAAA,OAAO,CAAA,EAAG,IAAI,CAAA,CAAA,EAAI,CAAC,CAAA,CAAA;AACrB;AC3GO,SAAS,0BAA0B,IAAA,EAAoD;AAC5F,EAAA,IAAI,IAAA,CAAK,sBAAsB,KAAA,EAAO;AACpC,IAAAA,gBAAA,CAAI,iBAAA,GAAoB,KAAA;AAAA,EAC1B;AAEA,EAAA,IAAI,eAAA,GAA2C,IAAA;AAC/C,EAAA,IAAI,iBAAA,GAAmC,IAAA;AAEvC,EAAA,SAAS,WAAA,GAAgC;AACvC,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAM,SAAA,GAAqC,EAAE,KAAA,EAAO,IAAA,CAAK,SAAS,MAAA,EAAO;AACzE,MAAA,IAAI,IAAA,CAAK,MAAA,KAAW,MAAA,EAAW,SAAA,CAAU,SAAS,IAAA,CAAK,MAAA;AACvD,MAAA,eAAA,GAAkBC,qBAAA,CAAS,oBAAA,EAAsB,IAAA,CAAK,KAAA,EAAO,SAAkB,CAAA;AAAA,IACjF;AACA,IAAA,OAAO,eAAA;AAAA,EACT;AAEA,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,OAAO,IAAA,CAAK,KAAA;AAAA,IACZ,aAAA,EAAe,KAAK,aAAA,IAAiB,MAAA;AAAA,IACrC,MAAA,EAAQ,KAAK,MAAA,IAAU,QAAA;AAAA,IACvB,UAAA,EAAY,IAAA;AAAA,IACZ,MAAM,MAAM,KAAA,EAA2C;AACrD,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,OAAO,EAAE,OAAA,EAAS,IAAI,SAAA,EAAW,IAAA,CAAK,aAAa,CAAA,EAAE;AAAA,MACvD;AACA,MAAA,MAAM,IAAA,GAAQ,MAAM,WAAA,EAAY;AAIhC,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,KAAA,EAAO,EAAE,OAAA,EAAS,MAAA,EAAQ,SAAA,EAAW,IAAA,EAAM,CAAA;AACrE,MAAA,MAAM,OAAO,MAAA,CAAO,IAAA;AACpB,MAAA,MAAM,YAAY,MAAA,CAAO,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,SAAS,CAAC,CAAA;AACpD,MAAA,IAAI,iBAAA,KAAsB,IAAA,IAAQ,iBAAA,KAAsB,SAAA,EAAW;AACjE,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,0CAAA,EAA6C,iBAAiB,CAAA,MAAA,EAAS,SAAS,CAAA,CAAE,CAAA;AAAA,MACpG;AACA,MAAA,iBAAA,GAAoB,SAAA;AACpB,MAAA,MAAM,UAA0B,EAAC;AACjC,MAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,KAAA,CAAM,MAAA,EAAQ,KAAK,CAAA,EAAG;AACxC,QAAA,OAAA,CAAQ,IAAA,CAAK,IAAI,YAAA,CAAa,IAAA,CAAK,MAAA,EAAQ,IAAA,CAAK,UAAA,GAAa,CAAA,GAAI,SAAA,GAAY,CAAA,EAAG,SAAS,CAAA,CAAE,OAAO,CAAA;AAAA,MACpG;AACA,MAAA,OAAO,EAAE,SAAS,SAAA,EAAU;AAAA,IAC9B;AAAA,GACF;AACA,EAAA,IAAI,IAAA,CAAK,SAAA,KAAc,MAAA,EAAW,OAAA,CAAQ,YAAY,IAAA,CAAK,SAAA;AAC3D,EAAA,OAAO,OAAA;AACT;;;ACvCO,IAAM,aAAA,GAAgB;AACtB,IAAM,uBAAA,GAA0B;;;ACNvC,eAAsB,KAAA,CAAM,QAAA,EAAkB,IAAA,GAAqB,EAAC,EAA+B;AACjG,EAAA,MAAM,MAAA,GAAS,cAAc,QAAA,EAAU,EAAE,MAAM,IAAA,CAAK,QAAA,IAAY,WAAW,CAAA;AAC3E,EAAA,MAAM,OAAA,GACJ,KAAK,OAAA,IACL,yBAAA;AAAA,IACE,KAAK,aAAA,KAAkB,MAAA,GACnB,EAAE,KAAA,EAAO,KAAK,KAAA,IAAS,aAAA,EAAe,aAAA,EAAe,IAAA,CAAK,eAAc,GACxE,EAAE,KAAA,EAAO,IAAA,CAAK,SAAS,aAAA;AAAc,GAC3C;AAEF,EAAA,MAAM,MAAA,GAAS,MAAM,OAAA,CAAQ,KAAA,CAAM,MAAA,CAAO,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,CAAC,CAAA;AAC5D,EAAA,IAAI,MAAA,CAAO,OAAA,CAAQ,MAAA,KAAW,MAAA,CAAO,MAAA,EAAQ;AAC3C,IAAA,MAAM,IAAI,MAAM,CAAA,iBAAA,EAAoB,MAAA,CAAO,QAAQ,MAAM,CAAA,aAAA,EAAgB,MAAA,CAAO,MAAM,CAAA,OAAA,CAAS,CAAA;AAAA,EACjG;AAEA,EAAA,MAAM,eAAA,GAAoC,MAAA,CAAO,GAAA,CAAI,CAAC,GAAG,CAAA,MAAO;AAAA,IAC9D,IAAI,CAAA,CAAE,EAAA;AAAA,IACN,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,MAAA,EAAQ,MAAA,CAAO,OAAA,CAAQ,CAAC;AAAA,GAC1B,CAAE,CAAA;AAEF,EAAA,MAAM,KAAA,GAAwB;AAAA,IAC5B,OAAO,OAAA,CAAQ,KAAA;AAAA,IACf,eAAe,OAAA,CAAQ,aAAA;AAAA,IACvB,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,QAAQ,OAAA,CAAQ,MAAA;AAAA,IAChB,YAAY,OAAA,CAAQ,UAAA;AAAA,IACpB,QAAA,EAAU,KAAK,QAAA,IAAY,SAAA;AAAA,IAC3B,MAAA,EAAQ;AAAA,GACV;AAEA,EAAA,OAAO,EAAE,aAAA,EAAe,CAAA,EAAG,MAAA,EAAQ,CAAC,KAAK,CAAA,EAAE;AAC7C;;;AC/BO,SAAS,cAAA,CACd,OAAA,EACA,WAAA,EACA,IAAA,GAAsB,EAAC,EACV;AACb,EAAA,MAAM,KAAA,GAAQ,SAAA,CAAU,OAAA,EAAS,IAAA,CAAK,KAAK,CAAA;AAC3C,EAAA,IAAI,CAAC,KAAA,EAAO,MAAM,IAAI,MAAM,mCAAmC,CAAA;AAC/D,EAAA,IAAI,WAAA,CAAY,MAAA,KAAW,KAAA,CAAM,SAAA,EAAW;AAC1C,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,uBAAA,EAA0B,WAAA,CAAY,MAAM,CAAA,sBAAA,EAAyB,KAAA,CAAM,KAAK,CAAA,EAAA,EAAK,KAAA,CAAM,SAAS,CAAA,CAAA,CAAG,CAAA;AAAA,EACzH;AAEA,EAAA,MAAM,CAAA,GAAI,KAAK,CAAA,IAAK,CAAA;AACpB,EAAA,MAAM,MAAA,GAAsB,KAAA,CAAM,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,IACnD,YAAY,KAAA,CAAM,KAAA;AAAA,IAClB,SAAS,CAAA,CAAE,EAAA;AAAA,IACX,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,OAAO,UAAA,CAAW,WAAA,EAAa,CAAA,CAAE,MAAA,EAAQ,MAAM,MAAM;AAAA,GACvD,CAAE,CAAA;AAGF,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,KAAW,WAAA,GAAc,CAAA,GAAI,EAAA;AACjD,EAAA,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,SAAS,CAAA,CAAE,KAAA,GAAQ,EAAE,KAAA,CAAM,CAAA;AACjD,EAAA,OAAO,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,CAAC,CAAA;AAC1B;AAEA,SAAS,SAAA,CAAU,SAA4B,SAAA,EAAgD;AAC7F,EAAA,IAAI,SAAA,SAAkB,OAAA,CAAQ,MAAA,CAAO,KAAK,CAAC,CAAA,KAAM,CAAA,CAAE,KAAA,KAAU,SAAS,CAAA;AACtE,EAAA,OAAO,OAAA,CAAQ,OAAO,CAAC,CAAA;AACzB;AAEA,SAAS,UAAA,CAAW,CAAA,EAAiB,CAAA,EAAiB,MAAA,EAAiC;AACrF,EAAA,IAAI,WAAW,WAAA,EAAa;AAC1B,IAAA,IAAI,GAAA,GAAM,CAAA;AACV,IAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,KAAK,CAAA,EAAG;AACpC,MAAA,MAAM,CAAA,GAAI,CAAA,CAAE,CAAC,CAAA,GAAK,EAAE,CAAC,CAAA;AACrB,MAAA,GAAA,IAAO,CAAA,GAAI,CAAA;AAAA,IACb;AACA,IAAA,OAAO,IAAA,CAAK,KAAK,GAAG,CAAA;AAAA,EACtB;AACA,EAAA,IAAI,GAAA,GAAM,CAAA;AACV,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,GAAA,IAAO,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACxD,EAAA,IAAI,MAAA,KAAW,OAAO,OAAO,GAAA;AAE7B,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,KAAK,CAAA,EAAG;AACpC,IAAA,EAAA,IAAM,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACjB,IAAA,EAAA,IAAM,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AAAA,EACnB;AACA,EAAA,MAAM,QAAQ,IAAA,CAAK,IAAA,CAAK,EAAE,CAAA,GAAI,IAAA,CAAK,KAAK,EAAE,CAAA;AAC1C,EAAA,OAAO,KAAA,KAAU,CAAA,GAAI,CAAA,GAAI,GAAA,GAAM,KAAA;AACjC;;;AClDA,IAAM,gBAAA,GAAmB,mDAAA;AAElB,SAAS,yBAAyB,IAAA,EAAmD;AAC1F,EAAA,MAAM,QAAQ,IAAA,CAAK,KAAA,IAAS,QAAQ,GAAA,CAAI,QAAA,IAAY,QAAQ,GAAA,CAAI,iBAAA;AAChE,EAAA,IAAI,CAAC,KAAA,EAAO;AACV,IAAA,MAAM,IAAI,MAAM,0EAA0E,CAAA;AAAA,EAC5F;AACA,EAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,gBAAA;AAChC,EAAA,MAAM,MAAM,CAAA,EAAG,OAAO,IAAI,SAAA,CAAU,IAAA,CAAK,KAAK,CAAC,CAAA,4BAAA,CAAA;AAE/C,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,OAAO,IAAA,CAAK,KAAA;AAAA,IACZ,aAAA,EAAe,KAAK,aAAA,IAAiB,MAAA;AAAA,IACrC,MAAA,EAAQ,KAAK,MAAA,IAAU,QAAA;AAAA,IACvB,UAAA,EAAY,IAAA;AAAA,IACZ,MAAM,MAAM,KAAA,EAA2C;AACrD,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,OAAO,EAAE,OAAA,EAAS,IAAI,SAAA,EAAW,IAAA,CAAK,aAAa,CAAA,EAAE;AAAA,MACvD;AACA,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,SAAA,CAAU,EAAE,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,EAAE,cAAA,EAAgB,IAAA,EAAK,EAAG,CAAA;AAChF,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,QAC3B,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB,kBAAA;AAAA,UAChB,aAAA,EAAe,UAAU,KAAK,CAAA;AAAA,SAChC;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,QAAA,MAAM,SAAS,MAAM,GAAA,CAAI,MAAK,CAAE,KAAA,CAAM,MAAM,EAAE,CAAA;AAC9C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iBAAA,EAAoB,GAAA,CAAI,MAAM,CAAA,KAAA,EAAQ,IAAA,CAAK,KAAK,CAAA,EAAA,EAAK,MAAA,IAAU,GAAA,CAAI,UAAU,CAAA,CAAE,CAAA;AAAA,MACjG;AACA,MAAA,MAAM,GAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,MAAA,MAAM,MAAA,GAAS,eAAA,CAAgB,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAChD,MAAA,MAAM,YAAY,MAAA,CAAO,CAAC,CAAA,EAAG,MAAA,IAAU,KAAK,SAAA,IAAa,CAAA;AACzD,MAAA,MAAM,OAAA,GAAU,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,UAAU,YAAA,CAAa,IAAA,CAAK,CAAC,CAAC,CAAC,CAAA;AACjE,MAAA,OAAO,EAAE,SAAS,SAAA,EAAU;AAAA,IAC9B;AAAA,GACF;AACA,EAAA,IAAI,IAAA,CAAK,SAAA,KAAc,MAAA,EAAW,OAAA,CAAQ,YAAY,IAAA,CAAK,SAAA;AAC3D,EAAA,OAAO,OAAA;AACT;AAWA,SAAS,eAAA,CAAgB,KAAc,aAAA,EAAmC;AACxE,EAAA,IAAI,CAAC,KAAA,CAAM,OAAA,CAAQ,GAAG,CAAA,EAAG;AACvB,IAAA,MAAM,IAAI,MAAM,2CAA2C,CAAA;AAAA,EAC7D;AACA,EAAA,IAAI,GAAA,CAAI,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAE9B,EAAA,MAAM,KAAA,GAAQ,IAAI,CAAC,CAAA;AACnB,EAAA,IAAI,OAAO,UAAU,QAAA,EAAU;AAC7B,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,yCAAA,EAA4C,aAAa,CAAA,CAAE,CAAA;AAAA,IAC7E;AACA,IAAA,OAAO,CAAC,GAAe,CAAA;AAAA,EACzB;AACA,EAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,KAAM,KAAA,CAAM,MAAA,KAAW,CAAA,IAAK,OAAO,KAAA,CAAM,CAAC,CAAA,KAAM,QAAA,CAAA,EAAW;AAChF,IAAA,OAAO,GAAA;AAAA,EACT;AACA,EAAA,IAAI,KAAA,CAAM,QAAQ,KAAK,CAAA,IAAK,MAAM,OAAA,CAAQ,KAAA,CAAM,CAAC,CAAC,CAAA,EAAG;AAEnD,IAAA,OAAQ,GAAA,CAAqB,IAAI,QAAQ,CAAA;AAAA,EAC3C;AACA,EAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AACjE;AAEA,SAAS,SAAS,MAAA,EAA8B;AAC9C,EAAA,IAAI,MAAA,CAAO,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AACjC,EAAA,MAAM,GAAA,GAAM,MAAA,CAAO,CAAC,CAAA,CAAG,MAAA;AACvB,EAAA,MAAM,MAAM,IAAI,KAAA,CAAc,GAAG,CAAA,CAAE,KAAK,CAAC,CAAA;AACzC,EAAA,KAAA,MAAW,KAAK,MAAA,EAAQ;AACtB,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,GAAA,EAAK,CAAA,IAAK,GAAG,GAAA,CAAI,CAAC,CAAA,IAAM,CAAA,CAAE,CAAC,CAAA;AAAA,EACjD;AACA,EAAA,KAAA,IAAS,CAAA,GAAI,GAAG,CAAA,GAAI,GAAA,EAAK,KAAK,CAAA,EAAG,GAAA,CAAI,CAAC,CAAA,IAAM,MAAA,CAAO,MAAA;AACnD,EAAA,OAAO,GAAA;AACT;AAEA,SAAS,UAAU,CAAA,EAA+B;AAChD,EAAA,IAAI,GAAA,GAAM,CAAA;AACV,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,GAAA,IAAO,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACxD,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,IAAA,CAAK,GAAG,CAAA;AAC1B,EAAA,IAAI,IAAA,KAAS,GAAG,OAAO,CAAA;AACvB,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,CAAA,CAAE,CAAC,CAAA,GAAI,CAAA,CAAE,CAAC,CAAA,GAAK,IAAA;AACrD,EAAA,OAAO,CAAA;AACT","file":"index.cjs","sourcesContent":["/**\n * Section-based markdown chunker.\n *\n * Splits a markdown document on ATX headings (`#`, `##`, ...). Each chunk\n * carries the byte offset and length into the original UTF-8 source so a\n * downstream consumer can map a vector hit back to the exact substring\n * without re-tokenising. Pre-heading content becomes a \"preamble\" chunk.\n */\n\nexport type ChunkingMode = 'document' | 'section' | 'paragraph';\n\nexport interface MarkdownChunk {\n  id: string;\n  textOffset: number;\n  textLength: number;\n  text: string;\n}\n\nexport interface ChunkOptions {\n  mode?: ChunkingMode;\n}\n\nconst HEADING = /^(#{1,6})\\s+(.+?)\\s*$/;\n\nexport function chunkMarkdown(markdown: string, opts: ChunkOptions = {}): MarkdownChunk[] {\n  const mode = opts.mode ?? 'section';\n  if (mode === 'document') {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  if (mode === 'paragraph') {\n    return paragraphChunks(markdown);\n  }\n  return sectionChunks(markdown);\n}\n\nfunction sectionChunks(markdown: string): MarkdownChunk[] {\n  const lines = splitWithOffsets(markdown);\n  const sections: MarkdownChunk[] = [];\n  let current: { id: string; start: number; end: number } | null = null;\n  const ids = new Set<string>();\n\n  function flush(end: number): void {\n    if (!current) return;\n    const text = markdown.slice(current.start, end);\n    if (text.trim().length === 0) {\n      current = null;\n      return;\n    }\n    sections.push({ id: current.id, textOffset: current.start, textLength: end - current.start, text });\n    current = null;\n  }\n\n  for (const line of lines) {\n    const match = HEADING.exec(line.text);\n    if (match) {\n      flush(line.offset);\n      const id = uniqueId(slugify(match[2] ?? `section-${sections.length + 1}`), ids);\n      ids.add(id);\n      current = { id, start: line.offset, end: line.offset + line.text.length };\n      continue;\n    }\n    if (current === null) {\n      const id = uniqueId('preamble', ids);\n      ids.add(id);\n      current = { id, start: line.offset, end: line.offset + line.text.length };\n    } else {\n      current.end = line.offset + line.text.length;\n    }\n  }\n  flush(markdown.length);\n\n  if (sections.length === 0) {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  return sections;\n}\n\nfunction paragraphChunks(markdown: string): MarkdownChunk[] {\n  const out: MarkdownChunk[] = [];\n  const ids = new Set<string>();\n  let cursor = 0;\n  let i = 0;\n  while (cursor < markdown.length) {\n    let end = markdown.indexOf('\\n\\n', cursor);\n    if (end === -1) end = markdown.length;\n    const text = markdown.slice(cursor, end);\n    if (text.trim().length > 0) {\n      const id = uniqueId(slugify(text.split('\\n')[0] ?? `p-${i}`), ids);\n      ids.add(id);\n      out.push({ id, textOffset: cursor, textLength: text.length, text });\n      i += 1;\n    }\n    cursor = end + 2;\n  }\n  if (out.length === 0) {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  return out;\n}\n\nfunction splitWithOffsets(s: string): { text: string; offset: number }[] {\n  const lines: { text: string; offset: number }[] = [];\n  let offset = 0;\n  for (const line of s.split('\\n')) {\n    const withNl = line + (offset + line.length < s.length ? '\\n' : '');\n    lines.push({ text: withNl, offset });\n    offset += withNl.length;\n  }\n  return lines;\n}\n\nfunction slugify(s: string): string {\n  return (\n    s\n      .toLowerCase()\n      .replace(/[^a-z0-9]+/g, '-')\n      .replace(/^-+|-+$/g, '')\n      .slice(0, 64) || 'section'\n  );\n}\n\nfunction uniqueId(base: string, taken: Set<string>): string {\n  if (!taken.has(base)) return base;\n  let n = 2;\n  while (taken.has(`${base}-${n}`)) n += 1;\n  return `${base}-${n}`;\n}\n","/**\n * transformers.js backend (browser + Node + Bun via @huggingface/transformers).\n *\n * Loads the model lazily and reuses the pipeline across calls. Models are\n * cached on disk by transformers.js itself (HF cache layout).\n */\n\nimport { pipeline, env } from '@huggingface/transformers';\nimport type { EmbeddingBackend, EmbeddingBackendOptions, EmbeddingMatrix } from './types.js';\n\ninterface TransformersBackendOptions extends EmbeddingBackendOptions {\n  /** Override Xenova quantisation. Defaults to fp32 for closest fidelity. */\n  dtype?: 'fp32' | 'fp16' | 'q8' | 'q4';\n  /** Force a backend device (e.g. 'cpu', 'gpu', 'wasm'). Defaults to auto. */\n  device?: 'cpu' | 'gpu' | 'wasm' | 'webgpu';\n  /** Allow remote model downloads. true by default. */\n  allowRemoteModels?: boolean;\n}\n\nexport function createTransformersBackend(opts: TransformersBackendOptions): EmbeddingBackend {\n  if (opts.allowRemoteModels === false) {\n    env.allowRemoteModels = false;\n  }\n\n  let pipelinePromise: Promise<unknown> | null = null;\n  let resolvedDimension: number | null = null;\n\n  function getPipeline(): Promise<unknown> {\n    if (!pipelinePromise) {\n      const modelOpts: Record<string, unknown> = { dtype: opts.dtype ?? 'fp32' };\n      if (opts.device !== undefined) modelOpts.device = opts.device;\n      pipelinePromise = pipeline('feature-extraction', opts.model, modelOpts as never);\n    }\n    return pipelinePromise;\n  }\n\n  const backend: EmbeddingBackend = {\n    model: opts.model,\n    modelRevision: opts.modelRevision ?? 'main',\n    metric: opts.metric ?? 'cosine',\n    normalized: true,\n    async embed(texts: string[]): Promise<EmbeddingMatrix> {\n      if (texts.length === 0) {\n        return { vectors: [], dimension: opts.dimension ?? 0 };\n      }\n      const pipe = (await getPipeline()) as (\n        texts: string[],\n        opts: { pooling: 'mean'; normalize: boolean },\n      ) => Promise<{ data: Float32Array; dims: number[] }>;\n      const tensor = await pipe(texts, { pooling: 'mean', normalize: true });\n      const data = tensor.data;\n      const dimension = tensor.dims[tensor.dims.length - 1] as number;\n      if (resolvedDimension !== null && resolvedDimension !== dimension) {\n        throw new Error(`Model emitted inconsistent dimension: had ${resolvedDimension}, now ${dimension}`);\n      }\n      resolvedDimension = dimension;\n      const vectors: Float32Array[] = [];\n      for (let i = 0; i < texts.length; i += 1) {\n        vectors.push(new Float32Array(data.buffer, data.byteOffset + i * dimension * 4, dimension).slice());\n      }\n      return { vectors, dimension };\n    },\n  };\n  if (opts.dimension !== undefined) backend.dimension = opts.dimension;\n  return backend;\n}\n","import type { EmbeddingMetric } from '@cvfile/sdk';\n\nexport interface EmbeddingMatrix {\n  vectors: Float32Array[];\n  dimension: number;\n}\n\nexport interface EmbeddingBackend {\n  model: string;\n  modelRevision: string;\n  /** Optional pre-declared dimension; the backend may override after first call. */\n  dimension?: number;\n  metric: EmbeddingMetric;\n  normalized: boolean;\n  embed(texts: string[]): Promise<EmbeddingMatrix>;\n}\n\nexport interface EmbeddingBackendOptions {\n  model: string;\n  modelRevision?: string;\n  /** Optional pre-known dimension (e.g. 1024 for bge-m3). */\n  dimension?: number;\n  metric?: EmbeddingMetric;\n}\n\n/** Recommended default per spec §5: BAAI BGE-M3, MIT, multilingual, 1024-dim. */\nexport const DEFAULT_MODEL = 'Xenova/bge-m3';\nexport const DEFAULT_MODEL_DIMENSION = 1024;\n","/**\n * High-level embed() API: markdown in, EmbeddingsPayload out, ready to drop\n * into pack({ embeddings: ... }).\n */\n\nimport type { EmbeddingChunk, EmbeddingSpace, EmbeddingsPayload } from '@cvfile/sdk';\nimport { chunkMarkdown, type ChunkingMode } from './chunk.js';\nimport { createTransformersBackend } from './transformers-backend.js';\nimport { DEFAULT_MODEL, type EmbeddingBackend } from './types.js';\n\nexport interface EmbedOptions {\n  /** HF model id; defaults to BGE-M3 (Xenova/bge-m3). */\n  model?: string;\n  /** Pinned model revision; recorded in the payload for reproducibility. */\n  modelRevision?: string;\n  /** Chunking strategy; default 'section'. */\n  chunking?: ChunkingMode;\n  /** Bring-your-own backend (e.g. an OpenAI/Voyage adapter). */\n  backend?: EmbeddingBackend;\n}\n\nexport async function embed(markdown: string, opts: EmbedOptions = {}): Promise<EmbeddingsPayload> {\n  const chunks = chunkMarkdown(markdown, { mode: opts.chunking ?? 'section' });\n  const backend =\n    opts.backend ??\n    createTransformersBackend(\n      opts.modelRevision !== undefined\n        ? { model: opts.model ?? DEFAULT_MODEL, modelRevision: opts.modelRevision }\n        : { model: opts.model ?? DEFAULT_MODEL },\n    );\n\n  const matrix = await backend.embed(chunks.map((c) => c.text));\n  if (matrix.vectors.length !== chunks.length) {\n    throw new Error(`Backend returned ${matrix.vectors.length} vectors for ${chunks.length} chunks`);\n  }\n\n  const embeddingChunks: EmbeddingChunk[] = chunks.map((c, i) => ({\n    id: c.id,\n    textOffset: c.textOffset,\n    textLength: c.textLength,\n    vector: matrix.vectors[i]!,\n  }));\n\n  const space: EmbeddingSpace = {\n    model: backend.model,\n    modelRevision: backend.modelRevision,\n    dimension: matrix.dimension,\n    metric: backend.metric,\n    normalized: backend.normalized,\n    chunking: opts.chunking ?? 'section',\n    chunks: embeddingChunks,\n  };\n\n  return { formatVersion: 1, spaces: [space] };\n}\n","/**\n * Pure similarity search over an EmbeddingsPayload. Vector input is the\n * caller's responsibility (encode the query with the same model that was\n * used to populate the space, ideally pulled from space.model).\n */\n\nimport type { EmbeddingMetric, EmbeddingSpace, EmbeddingsPayload } from '@cvfile/sdk';\n\nexport interface SearchHit {\n  spaceModel: string;\n  chunkId: string;\n  textOffset: number;\n  textLength: number;\n  score: number;\n}\n\nexport interface SearchOptions {\n  /** Restrict to a specific embedding space; default = first space. */\n  model?: string;\n  /** Top-k results; default 5. */\n  k?: number;\n}\n\nexport function searchSemantic(\n  payload: EmbeddingsPayload,\n  queryVector: Float32Array,\n  opts: SearchOptions = {},\n): SearchHit[] {\n  const space = pickSpace(payload, opts.model);\n  if (!space) throw new Error('No matching embedding space found');\n  if (queryVector.length !== space.dimension) {\n    throw new Error(`Query vector dimension ${queryVector.length} does not match space ${space.model} (${space.dimension})`);\n  }\n\n  const k = opts.k ?? 5;\n  const scored: SearchHit[] = space.chunks.map((c) => ({\n    spaceModel: space.model,\n    chunkId: c.id,\n    textOffset: c.textOffset,\n    textLength: c.textLength,\n    score: similarity(queryVector, c.vector, space.metric),\n  }));\n\n  // Higher score = better for cosine/dot; lower for euclidean.\n  const order = space.metric === 'euclidean' ? 1 : -1;\n  scored.sort((a, b) => order * (a.score - b.score));\n  return scored.slice(0, k);\n}\n\nfunction pickSpace(payload: EmbeddingsPayload, modelHint?: string): EmbeddingSpace | undefined {\n  if (modelHint) return payload.spaces.find((s) => s.model === modelHint);\n  return payload.spaces[0];\n}\n\nfunction similarity(a: Float32Array, b: Float32Array, metric: EmbeddingMetric): number {\n  if (metric === 'euclidean') {\n    let sum = 0;\n    for (let i = 0; i < a.length; i += 1) {\n      const d = a[i]! - b[i]!;\n      sum += d * d;\n    }\n    return Math.sqrt(sum);\n  }\n  let dot = 0;\n  for (let i = 0; i < a.length; i += 1) dot += a[i]! * b[i]!;\n  if (metric === 'dot') return dot;\n  // cosine: assume normalized vectors when produced by our backend\n  let na = 0;\n  let nb = 0;\n  for (let i = 0; i < a.length; i += 1) {\n    na += a[i]! * a[i]!;\n    nb += b[i]! * b[i]!;\n  }\n  const denom = Math.sqrt(na) * Math.sqrt(nb);\n  return denom === 0 ? 0 : dot / denom;\n}\n","/**\n * Hugging Face Inference API backend.\n *\n * Calls https://router.huggingface.co/hf-inference/models/<model>/pipeline/feature-extraction\n * with HF_TOKEN. For sentence-transformers models (BGE-M3, MiniLM, etc.) the\n * response is already mean-pooled per input — one vector per text. We\n * normalise client-side so cosine math is consistent across backends.\n */\n\nimport type { EmbeddingBackend, EmbeddingMatrix } from './types.js';\nimport type { EmbeddingMetric } from '@cvfile/sdk';\n\nexport interface HuggingFaceBackendOptions {\n  model: string;\n  /** HF token. Defaults to `process.env.HF_TOKEN`. */\n  token?: string;\n  /** Pinned revision; recorded in the payload. Default 'main'. */\n  modelRevision?: string;\n  /** Pre-known dimension. Optional; inferred from first response otherwise. */\n  dimension?: number;\n  metric?: EmbeddingMetric;\n  /** Override base URL (e.g. for self-hosted TEI). */\n  baseUrl?: string;\n}\n\nconst DEFAULT_BASE_URL = 'https://router.huggingface.co/hf-inference/models';\n\nexport function createHuggingFaceBackend(opts: HuggingFaceBackendOptions): EmbeddingBackend {\n  const token = opts.token ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_TOKEN;\n  if (!token) {\n    throw new Error('HF_TOKEN (or HUGGINGFACE_TOKEN) is required for the Hugging Face backend');\n  }\n  const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;\n  const url = `${baseUrl}/${encodeURI(opts.model)}/pipeline/feature-extraction`;\n\n  const backend: EmbeddingBackend = {\n    model: opts.model,\n    modelRevision: opts.modelRevision ?? 'main',\n    metric: opts.metric ?? 'cosine',\n    normalized: true,\n    async embed(texts: string[]): Promise<EmbeddingMatrix> {\n      if (texts.length === 0) {\n        return { vectors: [], dimension: opts.dimension ?? 0 };\n      }\n      const body = JSON.stringify({ inputs: texts, options: { wait_for_model: true } });\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          authorization: `Bearer ${token}`,\n        },\n        body,\n      });\n      if (!res.ok) {\n        const detail = await res.text().catch(() => '');\n        throw new Error(`HF Inference API ${res.status} for ${opts.model}: ${detail || res.statusText}`);\n      }\n      const raw = (await res.json()) as unknown;\n      const matrix = parseHfResponse(raw, texts.length);\n      const dimension = matrix[0]?.length ?? opts.dimension ?? 0;\n      const vectors = matrix.map((v) => normalize(Float32Array.from(v)));\n      return { vectors, dimension };\n    },\n  };\n  if (opts.dimension !== undefined) backend.dimension = opts.dimension;\n  return backend;\n}\n\n/**\n * Coerce the variety of shapes the HF Inference API returns into a flat\n * `number[][]`: one mean-pooled vector per input.\n *\n * Observed shapes:\n *   - sentence-transformers (BGE-M3, MiniLM): `[[...vec], [...vec]]`\n *   - feature-extraction without pooling: `[[[...token0], [...token1], ...], ...]`\n *   - single-input convenience form: `[...vec]`\n */\nfunction parseHfResponse(raw: unknown, expectedCount: number): number[][] {\n  if (!Array.isArray(raw)) {\n    throw new Error('HF Inference API: expected array response');\n  }\n  if (raw.length === 0) return [];\n\n  const first = raw[0];\n  if (typeof first === 'number') {\n    if (expectedCount !== 1) {\n      throw new Error(`HF Inference API: got 1 vector, expected ${expectedCount}`);\n    }\n    return [raw as number[]];\n  }\n  if (Array.isArray(first) && (first.length === 0 || typeof first[0] === 'number')) {\n    return raw as number[][];\n  }\n  if (Array.isArray(first) && Array.isArray(first[0])) {\n    // token-level embeddings: mean-pool per input\n    return (raw as number[][][]).map(meanPool);\n  }\n  throw new Error('HF Inference API: unrecognised response shape');\n}\n\nfunction meanPool(tokens: number[][]): number[] {\n  if (tokens.length === 0) return [];\n  const dim = tokens[0]!.length;\n  const out = new Array<number>(dim).fill(0);\n  for (const t of tokens) {\n    for (let i = 0; i < dim; i += 1) out[i]! += t[i]!;\n  }\n  for (let i = 0; i < dim; i += 1) out[i]! /= tokens.length;\n  return out;\n}\n\nfunction normalize(v: Float32Array): Float32Array {\n  let sum = 0;\n  for (let i = 0; i < v.length; i += 1) sum += v[i]! * v[i]!;\n  const norm = Math.sqrt(sum);\n  if (norm === 0) return v;\n  for (let i = 0; i < v.length; i += 1) v[i] = v[i]! / norm;\n  return v;\n}\n"]}

package/dist/index.d.cts

@@ -0,0 +1,125 @@
+import { EmbeddingMetric, EmbeddingsPayload } from '@cvfile/sdk';
+
+/**
+ * Section-based markdown chunker.
+ *
+ * Splits a markdown document on ATX headings (`#`, `##`, ...). Each chunk
+ * carries the byte offset and length into the original UTF-8 source so a
+ * downstream consumer can map a vector hit back to the exact substring
+ * without re-tokenising. Pre-heading content becomes a "preamble" chunk.
+ */
+type ChunkingMode = 'document' | 'section' | 'paragraph';
+interface MarkdownChunk {
+    id: string;
+    textOffset: number;
+    textLength: number;
+    text: string;
+}
+interface ChunkOptions {
+    mode?: ChunkingMode;
+}
+declare function chunkMarkdown(markdown: string, opts?: ChunkOptions): MarkdownChunk[];
+
+interface EmbeddingMatrix {
+    vectors: Float32Array[];
+    dimension: number;
+}
+interface EmbeddingBackend {
+    model: string;
+    modelRevision: string;
+    /** Optional pre-declared dimension; the backend may override after first call. */
+    dimension?: number;
+    metric: EmbeddingMetric;
+    normalized: boolean;
+    embed(texts: string[]): Promise<EmbeddingMatrix>;
+}
+interface EmbeddingBackendOptions {
+    model: string;
+    modelRevision?: string;
+    /** Optional pre-known dimension (e.g. 1024 for bge-m3). */
+    dimension?: number;
+    metric?: EmbeddingMetric;
+}
+/** Recommended default per spec §5: BAAI BGE-M3, MIT, multilingual, 1024-dim. */
+declare const DEFAULT_MODEL = "Xenova/bge-m3";
+declare const DEFAULT_MODEL_DIMENSION = 1024;
+
+/**
+ * High-level embed() API: markdown in, EmbeddingsPayload out, ready to drop
+ * into pack({ embeddings: ... }).
+ */
+
+interface EmbedOptions {
+    /** HF model id; defaults to BGE-M3 (Xenova/bge-m3). */
+    model?: string;
+    /** Pinned model revision; recorded in the payload for reproducibility. */
+    modelRevision?: string;
+    /** Chunking strategy; default 'section'. */
+    chunking?: ChunkingMode;
+    /** Bring-your-own backend (e.g. an OpenAI/Voyage adapter). */
+    backend?: EmbeddingBackend;
+}
+declare function embed(markdown: string, opts?: EmbedOptions): Promise<EmbeddingsPayload>;
+
+/**
+ * Pure similarity search over an EmbeddingsPayload. Vector input is the
+ * caller's responsibility (encode the query with the same model that was
+ * used to populate the space, ideally pulled from space.model).
+ */
+
+interface SearchHit {
+    spaceModel: string;
+    chunkId: string;
+    textOffset: number;
+    textLength: number;
+    score: number;
+}
+interface SearchOptions {
+    /** Restrict to a specific embedding space; default = first space. */
+    model?: string;
+    /** Top-k results; default 5. */
+    k?: number;
+}
+declare function searchSemantic(payload: EmbeddingsPayload, queryVector: Float32Array, opts?: SearchOptions): SearchHit[];
+
+/**
+ * transformers.js backend (browser + Node + Bun via @huggingface/transformers).
+ *
+ * Loads the model lazily and reuses the pipeline across calls. Models are
+ * cached on disk by transformers.js itself (HF cache layout).
+ */
+
+interface TransformersBackendOptions extends EmbeddingBackendOptions {
+    /** Override Xenova quantisation. Defaults to fp32 for closest fidelity. */
+    dtype?: 'fp32' | 'fp16' | 'q8' | 'q4';
+    /** Force a backend device (e.g. 'cpu', 'gpu', 'wasm'). Defaults to auto. */
+    device?: 'cpu' | 'gpu' | 'wasm' | 'webgpu';
+    /** Allow remote model downloads. true by default. */
+    allowRemoteModels?: boolean;
+}
+declare function createTransformersBackend(opts: TransformersBackendOptions): EmbeddingBackend;
+
+/**
+ * Hugging Face Inference API backend.
+ *
+ * Calls https://router.huggingface.co/hf-inference/models/<model>/pipeline/feature-extraction
+ * with HF_TOKEN. For sentence-transformers models (BGE-M3, MiniLM, etc.) the
+ * response is already mean-pooled per input — one vector per text. We
+ * normalise client-side so cosine math is consistent across backends.
+ */
+
+interface HuggingFaceBackendOptions {
+    model: string;
+    /** HF token. Defaults to `process.env.HF_TOKEN`. */
+    token?: string;
+    /** Pinned revision; recorded in the payload. Default 'main'. */
+    modelRevision?: string;
+    /** Pre-known dimension. Optional; inferred from first response otherwise. */
+    dimension?: number;
+    metric?: EmbeddingMetric;
+    /** Override base URL (e.g. for self-hosted TEI). */
+    baseUrl?: string;
+}
+declare function createHuggingFaceBackend(opts: HuggingFaceBackendOptions): EmbeddingBackend;
+
+export { type ChunkingMode, DEFAULT_MODEL, DEFAULT_MODEL_DIMENSION, type EmbedOptions, type EmbeddingBackend, type EmbeddingBackendOptions, type EmbeddingMatrix, type HuggingFaceBackendOptions, type MarkdownChunk, type SearchHit, type SearchOptions, chunkMarkdown, createHuggingFaceBackend, createTransformersBackend, embed, searchSemantic };

package/dist/index.d.ts

@@ -0,0 +1,125 @@
+import { EmbeddingMetric, EmbeddingsPayload } from '@cvfile/sdk';
+
+/**
+ * Section-based markdown chunker.
+ *
+ * Splits a markdown document on ATX headings (`#`, `##`, ...). Each chunk
+ * carries the byte offset and length into the original UTF-8 source so a
+ * downstream consumer can map a vector hit back to the exact substring
+ * without re-tokenising. Pre-heading content becomes a "preamble" chunk.
+ */
+type ChunkingMode = 'document' | 'section' | 'paragraph';
+interface MarkdownChunk {
+    id: string;
+    textOffset: number;
+    textLength: number;
+    text: string;
+}
+interface ChunkOptions {
+    mode?: ChunkingMode;
+}
+declare function chunkMarkdown(markdown: string, opts?: ChunkOptions): MarkdownChunk[];
+
+interface EmbeddingMatrix {
+    vectors: Float32Array[];
+    dimension: number;
+}
+interface EmbeddingBackend {
+    model: string;
+    modelRevision: string;
+    /** Optional pre-declared dimension; the backend may override after first call. */
+    dimension?: number;
+    metric: EmbeddingMetric;
+    normalized: boolean;
+    embed(texts: string[]): Promise<EmbeddingMatrix>;
+}
+interface EmbeddingBackendOptions {
+    model: string;
+    modelRevision?: string;
+    /** Optional pre-known dimension (e.g. 1024 for bge-m3). */
+    dimension?: number;
+    metric?: EmbeddingMetric;
+}
+/** Recommended default per spec §5: BAAI BGE-M3, MIT, multilingual, 1024-dim. */
+declare const DEFAULT_MODEL = "Xenova/bge-m3";
+declare const DEFAULT_MODEL_DIMENSION = 1024;
+
+/**
+ * High-level embed() API: markdown in, EmbeddingsPayload out, ready to drop
+ * into pack({ embeddings: ... }).
+ */
+
+interface EmbedOptions {
+    /** HF model id; defaults to BGE-M3 (Xenova/bge-m3). */
+    model?: string;
+    /** Pinned model revision; recorded in the payload for reproducibility. */
+    modelRevision?: string;
+    /** Chunking strategy; default 'section'. */
+    chunking?: ChunkingMode;
+    /** Bring-your-own backend (e.g. an OpenAI/Voyage adapter). */
+    backend?: EmbeddingBackend;
+}
+declare function embed(markdown: string, opts?: EmbedOptions): Promise<EmbeddingsPayload>;
+
+/**
+ * Pure similarity search over an EmbeddingsPayload. Vector input is the
+ * caller's responsibility (encode the query with the same model that was
+ * used to populate the space, ideally pulled from space.model).
+ */
+
+interface SearchHit {
+    spaceModel: string;
+    chunkId: string;
+    textOffset: number;
+    textLength: number;
+    score: number;
+}
+interface SearchOptions {
+    /** Restrict to a specific embedding space; default = first space. */
+    model?: string;
+    /** Top-k results; default 5. */
+    k?: number;
+}
+declare function searchSemantic(payload: EmbeddingsPayload, queryVector: Float32Array, opts?: SearchOptions): SearchHit[];
+
+/**
+ * transformers.js backend (browser + Node + Bun via @huggingface/transformers).
+ *
+ * Loads the model lazily and reuses the pipeline across calls. Models are
+ * cached on disk by transformers.js itself (HF cache layout).
+ */
+
+interface TransformersBackendOptions extends EmbeddingBackendOptions {
+    /** Override Xenova quantisation. Defaults to fp32 for closest fidelity. */
+    dtype?: 'fp32' | 'fp16' | 'q8' | 'q4';
+    /** Force a backend device (e.g. 'cpu', 'gpu', 'wasm'). Defaults to auto. */
+    device?: 'cpu' | 'gpu' | 'wasm' | 'webgpu';
+    /** Allow remote model downloads. true by default. */
+    allowRemoteModels?: boolean;
+}
+declare function createTransformersBackend(opts: TransformersBackendOptions): EmbeddingBackend;
+
+/**
+ * Hugging Face Inference API backend.
+ *
+ * Calls https://router.huggingface.co/hf-inference/models/<model>/pipeline/feature-extraction
+ * with HF_TOKEN. For sentence-transformers models (BGE-M3, MiniLM, etc.) the
+ * response is already mean-pooled per input — one vector per text. We
+ * normalise client-side so cosine math is consistent across backends.
+ */
+
+interface HuggingFaceBackendOptions {
+    model: string;
+    /** HF token. Defaults to `process.env.HF_TOKEN`. */
+    token?: string;
+    /** Pinned revision; recorded in the payload. Default 'main'. */
+    modelRevision?: string;
+    /** Pre-known dimension. Optional; inferred from first response otherwise. */
+    dimension?: number;
+    metric?: EmbeddingMetric;
+    /** Override base URL (e.g. for self-hosted TEI). */
+    baseUrl?: string;
+}
+declare function createHuggingFaceBackend(opts: HuggingFaceBackendOptions): EmbeddingBackend;
+
+export { type ChunkingMode, DEFAULT_MODEL, DEFAULT_MODEL_DIMENSION, type EmbedOptions, type EmbeddingBackend, type EmbeddingBackendOptions, type EmbeddingMatrix, type HuggingFaceBackendOptions, type MarkdownChunk, type SearchHit, type SearchOptions, chunkMarkdown, createHuggingFaceBackend, createTransformersBackend, embed, searchSemantic };

package/dist/index.js

@@ -0,0 +1,295 @@
+import { env, pipeline } from '@huggingface/transformers';
+
+// src/chunk.ts
+var HEADING = /^(#{1,6})\s+(.+?)\s*$/;
+function chunkMarkdown(markdown, opts = {}) {
+  const mode = opts.mode ?? "section";
+  if (mode === "document") {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  if (mode === "paragraph") {
+    return paragraphChunks(markdown);
+  }
+  return sectionChunks(markdown);
+}
+function sectionChunks(markdown) {
+  const lines = splitWithOffsets(markdown);
+  const sections = [];
+  let current = null;
+  const ids = /* @__PURE__ */ new Set();
+  function flush(end) {
+    if (!current) return;
+    const text = markdown.slice(current.start, end);
+    if (text.trim().length === 0) {
+      current = null;
+      return;
+    }
+    sections.push({ id: current.id, textOffset: current.start, textLength: end - current.start, text });
+    current = null;
+  }
+  for (const line of lines) {
+    const match = HEADING.exec(line.text);
+    if (match) {
+      flush(line.offset);
+      const id = uniqueId(slugify(match[2] ?? `section-${sections.length + 1}`), ids);
+      ids.add(id);
+      current = { id, start: line.offset, end: line.offset + line.text.length };
+      continue;
+    }
+    if (current === null) {
+      const id = uniqueId("preamble", ids);
+      ids.add(id);
+      current = { id, start: line.offset, end: line.offset + line.text.length };
+    } else {
+      current.end = line.offset + line.text.length;
+    }
+  }
+  flush(markdown.length);
+  if (sections.length === 0) {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  return sections;
+}
+function paragraphChunks(markdown) {
+  const out = [];
+  const ids = /* @__PURE__ */ new Set();
+  let cursor = 0;
+  let i = 0;
+  while (cursor < markdown.length) {
+    let end = markdown.indexOf("\n\n", cursor);
+    if (end === -1) end = markdown.length;
+    const text = markdown.slice(cursor, end);
+    if (text.trim().length > 0) {
+      const id = uniqueId(slugify(text.split("\n")[0] ?? `p-${i}`), ids);
+      ids.add(id);
+      out.push({ id, textOffset: cursor, textLength: text.length, text });
+      i += 1;
+    }
+    cursor = end + 2;
+  }
+  if (out.length === 0) {
+    return [{ id: "document", textOffset: 0, textLength: markdown.length, text: markdown }];
+  }
+  return out;
+}
+function splitWithOffsets(s) {
+  const lines = [];
+  let offset = 0;
+  for (const line of s.split("\n")) {
+    const withNl = line + (offset + line.length < s.length ? "\n" : "");
+    lines.push({ text: withNl, offset });
+    offset += withNl.length;
+  }
+  return lines;
+}
+function slugify(s) {
+  return s.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 64) || "section";
+}
+function uniqueId(base, taken) {
+  if (!taken.has(base)) return base;
+  let n = 2;
+  while (taken.has(`${base}-${n}`)) n += 1;
+  return `${base}-${n}`;
+}
+function createTransformersBackend(opts) {
+  if (opts.allowRemoteModels === false) {
+    env.allowRemoteModels = false;
+  }
+  let pipelinePromise = null;
+  let resolvedDimension = null;
+  function getPipeline() {
+    if (!pipelinePromise) {
+      const modelOpts = { dtype: opts.dtype ?? "fp32" };
+      if (opts.device !== void 0) modelOpts.device = opts.device;
+      pipelinePromise = pipeline("feature-extraction", opts.model, modelOpts);
+    }
+    return pipelinePromise;
+  }
+  const backend = {
+    model: opts.model,
+    modelRevision: opts.modelRevision ?? "main",
+    metric: opts.metric ?? "cosine",
+    normalized: true,
+    async embed(texts) {
+      if (texts.length === 0) {
+        return { vectors: [], dimension: opts.dimension ?? 0 };
+      }
+      const pipe = await getPipeline();
+      const tensor = await pipe(texts, { pooling: "mean", normalize: true });
+      const data = tensor.data;
+      const dimension = tensor.dims[tensor.dims.length - 1];
+      if (resolvedDimension !== null && resolvedDimension !== dimension) {
+        throw new Error(`Model emitted inconsistent dimension: had ${resolvedDimension}, now ${dimension}`);
+      }
+      resolvedDimension = dimension;
+      const vectors = [];
+      for (let i = 0; i < texts.length; i += 1) {
+        vectors.push(new Float32Array(data.buffer, data.byteOffset + i * dimension * 4, dimension).slice());
+      }
+      return { vectors, dimension };
+    }
+  };
+  if (opts.dimension !== void 0) backend.dimension = opts.dimension;
+  return backend;
+}
+
+// src/types.ts
+var DEFAULT_MODEL = "Xenova/bge-m3";
+var DEFAULT_MODEL_DIMENSION = 1024;
+
+// src/embed.ts
+async function embed(markdown, opts = {}) {
+  const chunks = chunkMarkdown(markdown, { mode: opts.chunking ?? "section" });
+  const backend = opts.backend ?? createTransformersBackend(
+    opts.modelRevision !== void 0 ? { model: opts.model ?? DEFAULT_MODEL, modelRevision: opts.modelRevision } : { model: opts.model ?? DEFAULT_MODEL }
+  );
+  const matrix = await backend.embed(chunks.map((c) => c.text));
+  if (matrix.vectors.length !== chunks.length) {
+    throw new Error(`Backend returned ${matrix.vectors.length} vectors for ${chunks.length} chunks`);
+  }
+  const embeddingChunks = chunks.map((c, i) => ({
+    id: c.id,
+    textOffset: c.textOffset,
+    textLength: c.textLength,
+    vector: matrix.vectors[i]
+  }));
+  const space = {
+    model: backend.model,
+    modelRevision: backend.modelRevision,
+    dimension: matrix.dimension,
+    metric: backend.metric,
+    normalized: backend.normalized,
+    chunking: opts.chunking ?? "section",
+    chunks: embeddingChunks
+  };
+  return { formatVersion: 1, spaces: [space] };
+}
+
+// src/search.ts
+function searchSemantic(payload, queryVector, opts = {}) {
+  const space = pickSpace(payload, opts.model);
+  if (!space) throw new Error("No matching embedding space found");
+  if (queryVector.length !== space.dimension) {
+    throw new Error(`Query vector dimension ${queryVector.length} does not match space ${space.model} (${space.dimension})`);
+  }
+  const k = opts.k ?? 5;
+  const scored = space.chunks.map((c) => ({
+    spaceModel: space.model,
+    chunkId: c.id,
+    textOffset: c.textOffset,
+    textLength: c.textLength,
+    score: similarity(queryVector, c.vector, space.metric)
+  }));
+  const order = space.metric === "euclidean" ? 1 : -1;
+  scored.sort((a, b) => order * (a.score - b.score));
+  return scored.slice(0, k);
+}
+function pickSpace(payload, modelHint) {
+  if (modelHint) return payload.spaces.find((s) => s.model === modelHint);
+  return payload.spaces[0];
+}
+function similarity(a, b, metric) {
+  if (metric === "euclidean") {
+    let sum = 0;
+    for (let i = 0; i < a.length; i += 1) {
+      const d = a[i] - b[i];
+      sum += d * d;
+    }
+    return Math.sqrt(sum);
+  }
+  let dot = 0;
+  for (let i = 0; i < a.length; i += 1) dot += a[i] * b[i];
+  if (metric === "dot") return dot;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    na += a[i] * a[i];
+    nb += b[i] * b[i];
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+// src/huggingface-backend.ts
+var DEFAULT_BASE_URL = "https://router.huggingface.co/hf-inference/models";
+function createHuggingFaceBackend(opts) {
+  const token = opts.token ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_TOKEN;
+  if (!token) {
+    throw new Error("HF_TOKEN (or HUGGINGFACE_TOKEN) is required for the Hugging Face backend");
+  }
+  const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;
+  const url = `${baseUrl}/${encodeURI(opts.model)}/pipeline/feature-extraction`;
+  const backend = {
+    model: opts.model,
+    modelRevision: opts.modelRevision ?? "main",
+    metric: opts.metric ?? "cosine",
+    normalized: true,
+    async embed(texts) {
+      if (texts.length === 0) {
+        return { vectors: [], dimension: opts.dimension ?? 0 };
+      }
+      const body = JSON.stringify({ inputs: texts, options: { wait_for_model: true } });
+      const res = await fetch(url, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${token}`
+        },
+        body
+      });
+      if (!res.ok) {
+        const detail = await res.text().catch(() => "");
+        throw new Error(`HF Inference API ${res.status} for ${opts.model}: ${detail || res.statusText}`);
+      }
+      const raw = await res.json();
+      const matrix = parseHfResponse(raw, texts.length);
+      const dimension = matrix[0]?.length ?? opts.dimension ?? 0;
+      const vectors = matrix.map((v) => normalize(Float32Array.from(v)));
+      return { vectors, dimension };
+    }
+  };
+  if (opts.dimension !== void 0) backend.dimension = opts.dimension;
+  return backend;
+}
+function parseHfResponse(raw, expectedCount) {
+  if (!Array.isArray(raw)) {
+    throw new Error("HF Inference API: expected array response");
+  }
+  if (raw.length === 0) return [];
+  const first = raw[0];
+  if (typeof first === "number") {
+    if (expectedCount !== 1) {
+      throw new Error(`HF Inference API: got 1 vector, expected ${expectedCount}`);
+    }
+    return [raw];
+  }
+  if (Array.isArray(first) && (first.length === 0 || typeof first[0] === "number")) {
+    return raw;
+  }
+  if (Array.isArray(first) && Array.isArray(first[0])) {
+    return raw.map(meanPool);
+  }
+  throw new Error("HF Inference API: unrecognised response shape");
+}
+function meanPool(tokens) {
+  if (tokens.length === 0) return [];
+  const dim = tokens[0].length;
+  const out = new Array(dim).fill(0);
+  for (const t of tokens) {
+    for (let i = 0; i < dim; i += 1) out[i] += t[i];
+  }
+  for (let i = 0; i < dim; i += 1) out[i] /= tokens.length;
+  return out;
+}
+function normalize(v) {
+  let sum = 0;
+  for (let i = 0; i < v.length; i += 1) sum += v[i] * v[i];
+  const norm = Math.sqrt(sum);
+  if (norm === 0) return v;
+  for (let i = 0; i < v.length; i += 1) v[i] = v[i] / norm;
+  return v;
+}
+
+export { DEFAULT_MODEL, DEFAULT_MODEL_DIMENSION, chunkMarkdown, createHuggingFaceBackend, createTransformersBackend, embed, searchSemantic };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/chunk.ts","../src/transformers-backend.ts","../src/types.ts","../src/embed.ts","../src/search.ts","../src/huggingface-backend.ts"],"names":[],"mappings":";;;AAsBA,IAAM,OAAA,GAAU,uBAAA;AAET,SAAS,aAAA,CAAc,QAAA,EAAkB,IAAA,GAAqB,EAAC,EAAoB;AACxF,EAAA,MAAM,IAAA,GAAO,KAAK,IAAA,IAAQ,SAAA;AAC1B,EAAA,IAAI,SAAS,UAAA,EAAY;AACvB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,IAAI,SAAS,WAAA,EAAa;AACxB,IAAA,OAAO,gBAAgB,QAAQ,CAAA;AAAA,EACjC;AACA,EAAA,OAAO,cAAc,QAAQ,CAAA;AAC/B;AAEA,SAAS,cAAc,QAAA,EAAmC;AACxD,EAAA,MAAM,KAAA,GAAQ,iBAAiB,QAAQ,CAAA;AACvC,EAAA,MAAM,WAA4B,EAAC;AACnC,EAAA,IAAI,OAAA,GAA6D,IAAA;AACjE,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAE5B,EAAA,SAAS,MAAM,GAAA,EAAmB;AAChC,IAAA,IAAI,CAAC,OAAA,EAAS;AACd,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,KAAA,CAAM,OAAA,CAAQ,OAAO,GAAG,CAAA;AAC9C,IAAA,IAAI,IAAA,CAAK,IAAA,EAAK,CAAE,MAAA,KAAW,CAAA,EAAG;AAC5B,MAAA,OAAA,GAAU,IAAA;AACV,MAAA;AAAA,IACF;AACA,IAAA,QAAA,CAAS,IAAA,CAAK,EAAE,EAAA,EAAI,OAAA,CAAQ,EAAA,EAAI,UAAA,EAAY,OAAA,CAAQ,KAAA,EAAO,UAAA,EAAY,GAAA,GAAM,OAAA,CAAQ,KAAA,EAAO,MAAM,CAAA;AAClG,IAAA,OAAA,GAAU,IAAA;AAAA,EACZ;AAEA,EAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,IAAA,MAAM,KAAA,GAAQ,OAAA,CAAQ,IAAA,CAAK,IAAA,CAAK,IAAI,CAAA;AACpC,IAAA,IAAI,KAAA,EAAO;AACT,MAAA,KAAA,CAAM,KAAK,MAAM,CAAA;AACjB,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,OAAA,CAAQ,KAAA,CAAM,CAAC,CAAA,IAAK,CAAA,QAAA,EAAW,QAAA,CAAS,MAAA,GAAS,CAAC,CAAA,CAAE,CAAA,EAAG,GAAG,CAAA;AAC9E,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,OAAA,GAAU,EAAE,EAAA,EAAI,KAAA,EAAO,IAAA,CAAK,MAAA,EAAQ,KAAK,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA,EAAO;AACxE,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAY,IAAA,EAAM;AACpB,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,UAAA,EAAY,GAAG,CAAA;AACnC,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,OAAA,GAAU,EAAE,EAAA,EAAI,KAAA,EAAO,IAAA,CAAK,MAAA,EAAQ,KAAK,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA,EAAO;AAAA,IAC1E,CAAA,MAAO;AACL,MAAA,OAAA,CAAQ,GAAA,GAAM,IAAA,CAAK,MAAA,GAAS,IAAA,CAAK,IAAA,CAAK,MAAA;AAAA,IACxC;AAAA,EACF;AACA,EAAA,KAAA,CAAM,SAAS,MAAM,CAAA;AAErB,EAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,QAAA;AACT;AAEA,SAAS,gBAAgB,QAAA,EAAmC;AAC1D,EAAA,MAAM,MAAuB,EAAC;AAC9B,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAC5B,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,OAAO,MAAA,GAAS,SAAS,MAAA,EAAQ;AAC/B,IAAA,IAAI,GAAA,GAAM,QAAA,CAAS,OAAA,CAAQ,MAAA,EAAQ,MAAM,CAAA;AACzC,IAAA,IAAI,GAAA,KAAQ,EAAA,EAAI,GAAA,GAAM,QAAA,CAAS,MAAA;AAC/B,IAAA,MAAM,IAAA,GAAO,QAAA,CAAS,KAAA,CAAM,MAAA,EAAQ,GAAG,CAAA;AACvC,IAAA,IAAI,IAAA,CAAK,IAAA,EAAK,CAAE,MAAA,GAAS,CAAA,EAAG;AAC1B,MAAA,MAAM,EAAA,GAAK,QAAA,CAAS,OAAA,CAAQ,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA,CAAE,CAAC,CAAA,IAAK,CAAA,EAAA,EAAK,CAAC,CAAA,CAAE,GAAG,GAAG,CAAA;AACjE,MAAA,GAAA,CAAI,IAAI,EAAE,CAAA;AACV,MAAA,GAAA,CAAI,IAAA,CAAK,EAAE,EAAA,EAAI,UAAA,EAAY,QAAQ,UAAA,EAAY,IAAA,CAAK,MAAA,EAAQ,IAAA,EAAM,CAAA;AAClE,MAAA,CAAA,IAAK,CAAA;AAAA,IACP;AACA,IAAA,MAAA,GAAS,GAAA,GAAM,CAAA;AAAA,EACjB;AACA,EAAA,IAAI,GAAA,CAAI,WAAW,CAAA,EAAG;AACpB,IAAA,OAAO,CAAC,EAAE,EAAA,EAAI,UAAA,EAAY,UAAA,EAAY,CAAA,EAAG,UAAA,EAAY,QAAA,CAAS,MAAA,EAAQ,IAAA,EAAM,QAAA,EAAU,CAAA;AAAA,EACxF;AACA,EAAA,OAAO,GAAA;AACT;AAEA,SAAS,iBAAiB,CAAA,EAA+C;AACvE,EAAA,MAAM,QAA4C,EAAC;AACnD,EAAA,IAAI,MAAA,GAAS,CAAA;AACb,EAAA,KAAA,MAAW,IAAA,IAAQ,CAAA,CAAE,KAAA,CAAM,IAAI,CAAA,EAAG;AAChC,IAAA,MAAM,SAAS,IAAA,IAAQ,MAAA,GAAS,KAAK,MAAA,GAAS,CAAA,CAAE,SAAS,IAAA,GAAO,EAAA,CAAA;AAChE,IAAA,KAAA,CAAM,IAAA,CAAK,EAAE,IAAA,EAAM,MAAA,EAAQ,QAAQ,CAAA;AACnC,IAAA,MAAA,IAAU,MAAA,CAAO,MAAA;AAAA,EACnB;AACA,EAAA,OAAO,KAAA;AACT;AAEA,SAAS,QAAQ,CAAA,EAAmB;AAClC,EAAA,OACE,CAAA,CACG,WAAA,EAAY,CACZ,OAAA,CAAQ,eAAe,GAAG,CAAA,CAC1B,OAAA,CAAQ,UAAA,EAAY,EAAE,CAAA,CACtB,KAAA,CAAM,CAAA,EAAG,EAAE,CAAA,IAAK,SAAA;AAEvB;AAEA,SAAS,QAAA,CAAS,MAAc,KAAA,EAA4B;AAC1D,EAAA,IAAI,CAAC,KAAA,CAAM,GAAA,CAAI,IAAI,GAAG,OAAO,IAAA;AAC7B,EAAA,IAAI,CAAA,GAAI,CAAA;AACR,EAAA,OAAO,KAAA,CAAM,IAAI,CAAA,EAAG,IAAI,IAAI,CAAC,CAAA,CAAE,GAAG,CAAA,IAAK,CAAA;AACvC,EAAA,OAAO,CAAA,EAAG,IAAI,CAAA,CAAA,EAAI,CAAC,CAAA,CAAA;AACrB;AC3GO,SAAS,0BAA0B,IAAA,EAAoD;AAC5F,EAAA,IAAI,IAAA,CAAK,sBAAsB,KAAA,EAAO;AACpC,IAAA,GAAA,CAAI,iBAAA,GAAoB,KAAA;AAAA,EAC1B;AAEA,EAAA,IAAI,eAAA,GAA2C,IAAA;AAC/C,EAAA,IAAI,iBAAA,GAAmC,IAAA;AAEvC,EAAA,SAAS,WAAA,GAAgC;AACvC,IAAA,IAAI,CAAC,eAAA,EAAiB;AACpB,MAAA,MAAM,SAAA,GAAqC,EAAE,KAAA,EAAO,IAAA,CAAK,SAAS,MAAA,EAAO;AACzE,MAAA,IAAI,IAAA,CAAK,MAAA,KAAW,MAAA,EAAW,SAAA,CAAU,SAAS,IAAA,CAAK,MAAA;AACvD,MAAA,eAAA,GAAkB,QAAA,CAAS,oBAAA,EAAsB,IAAA,CAAK,KAAA,EAAO,SAAkB,CAAA;AAAA,IACjF;AACA,IAAA,OAAO,eAAA;AAAA,EACT;AAEA,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,OAAO,IAAA,CAAK,KAAA;AAAA,IACZ,aAAA,EAAe,KAAK,aAAA,IAAiB,MAAA;AAAA,IACrC,MAAA,EAAQ,KAAK,MAAA,IAAU,QAAA;AAAA,IACvB,UAAA,EAAY,IAAA;AAAA,IACZ,MAAM,MAAM,KAAA,EAA2C;AACrD,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,OAAO,EAAE,OAAA,EAAS,IAAI,SAAA,EAAW,IAAA,CAAK,aAAa,CAAA,EAAE;AAAA,MACvD;AACA,MAAA,MAAM,IAAA,GAAQ,MAAM,WAAA,EAAY;AAIhC,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,KAAA,EAAO,EAAE,OAAA,EAAS,MAAA,EAAQ,SAAA,EAAW,IAAA,EAAM,CAAA;AACrE,MAAA,MAAM,OAAO,MAAA,CAAO,IAAA;AACpB,MAAA,MAAM,YAAY,MAAA,CAAO,IAAA,CAAK,MAAA,CAAO,IAAA,CAAK,SAAS,CAAC,CAAA;AACpD,MAAA,IAAI,iBAAA,KAAsB,IAAA,IAAQ,iBAAA,KAAsB,SAAA,EAAW;AACjE,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,0CAAA,EAA6C,iBAAiB,CAAA,MAAA,EAAS,SAAS,CAAA,CAAE,CAAA;AAAA,MACpG;AACA,MAAA,iBAAA,GAAoB,SAAA;AACpB,MAAA,MAAM,UAA0B,EAAC;AACjC,MAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,KAAA,CAAM,MAAA,EAAQ,KAAK,CAAA,EAAG;AACxC,QAAA,OAAA,CAAQ,IAAA,CAAK,IAAI,YAAA,CAAa,IAAA,CAAK,MAAA,EAAQ,IAAA,CAAK,UAAA,GAAa,CAAA,GAAI,SAAA,GAAY,CAAA,EAAG,SAAS,CAAA,CAAE,OAAO,CAAA;AAAA,MACpG;AACA,MAAA,OAAO,EAAE,SAAS,SAAA,EAAU;AAAA,IAC9B;AAAA,GACF;AACA,EAAA,IAAI,IAAA,CAAK,SAAA,KAAc,MAAA,EAAW,OAAA,CAAQ,YAAY,IAAA,CAAK,SAAA;AAC3D,EAAA,OAAO,OAAA;AACT;;;ACvCO,IAAM,aAAA,GAAgB;AACtB,IAAM,uBAAA,GAA0B;;;ACNvC,eAAsB,KAAA,CAAM,QAAA,EAAkB,IAAA,GAAqB,EAAC,EAA+B;AACjG,EAAA,MAAM,MAAA,GAAS,cAAc,QAAA,EAAU,EAAE,MAAM,IAAA,CAAK,QAAA,IAAY,WAAW,CAAA;AAC3E,EAAA,MAAM,OAAA,GACJ,KAAK,OAAA,IACL,yBAAA;AAAA,IACE,KAAK,aAAA,KAAkB,MAAA,GACnB,EAAE,KAAA,EAAO,KAAK,KAAA,IAAS,aAAA,EAAe,aAAA,EAAe,IAAA,CAAK,eAAc,GACxE,EAAE,KAAA,EAAO,IAAA,CAAK,SAAS,aAAA;AAAc,GAC3C;AAEF,EAAA,MAAM,MAAA,GAAS,MAAM,OAAA,CAAQ,KAAA,CAAM,MAAA,CAAO,IAAI,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,CAAC,CAAA;AAC5D,EAAA,IAAI,MAAA,CAAO,OAAA,CAAQ,MAAA,KAAW,MAAA,CAAO,MAAA,EAAQ;AAC3C,IAAA,MAAM,IAAI,MAAM,CAAA,iBAAA,EAAoB,MAAA,CAAO,QAAQ,MAAM,CAAA,aAAA,EAAgB,MAAA,CAAO,MAAM,CAAA,OAAA,CAAS,CAAA;AAAA,EACjG;AAEA,EAAA,MAAM,eAAA,GAAoC,MAAA,CAAO,GAAA,CAAI,CAAC,GAAG,CAAA,MAAO;AAAA,IAC9D,IAAI,CAAA,CAAE,EAAA;AAAA,IACN,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,MAAA,EAAQ,MAAA,CAAO,OAAA,CAAQ,CAAC;AAAA,GAC1B,CAAE,CAAA;AAEF,EAAA,MAAM,KAAA,GAAwB;AAAA,IAC5B,OAAO,OAAA,CAAQ,KAAA;AAAA,IACf,eAAe,OAAA,CAAQ,aAAA;AAAA,IACvB,WAAW,MAAA,CAAO,SAAA;AAAA,IAClB,QAAQ,OAAA,CAAQ,MAAA;AAAA,IAChB,YAAY,OAAA,CAAQ,UAAA;AAAA,IACpB,QAAA,EAAU,KAAK,QAAA,IAAY,SAAA;AAAA,IAC3B,MAAA,EAAQ;AAAA,GACV;AAEA,EAAA,OAAO,EAAE,aAAA,EAAe,CAAA,EAAG,MAAA,EAAQ,CAAC,KAAK,CAAA,EAAE;AAC7C;;;AC/BO,SAAS,cAAA,CACd,OAAA,EACA,WAAA,EACA,IAAA,GAAsB,EAAC,EACV;AACb,EAAA,MAAM,KAAA,GAAQ,SAAA,CAAU,OAAA,EAAS,IAAA,CAAK,KAAK,CAAA;AAC3C,EAAA,IAAI,CAAC,KAAA,EAAO,MAAM,IAAI,MAAM,mCAAmC,CAAA;AAC/D,EAAA,IAAI,WAAA,CAAY,MAAA,KAAW,KAAA,CAAM,SAAA,EAAW;AAC1C,IAAA,MAAM,IAAI,KAAA,CAAM,CAAA,uBAAA,EAA0B,WAAA,CAAY,MAAM,CAAA,sBAAA,EAAyB,KAAA,CAAM,KAAK,CAAA,EAAA,EAAK,KAAA,CAAM,SAAS,CAAA,CAAA,CAAG,CAAA;AAAA,EACzH;AAEA,EAAA,MAAM,CAAA,GAAI,KAAK,CAAA,IAAK,CAAA;AACpB,EAAA,MAAM,MAAA,GAAsB,KAAA,CAAM,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,MAAO;AAAA,IACnD,YAAY,KAAA,CAAM,KAAA;AAAA,IAClB,SAAS,CAAA,CAAE,EAAA;AAAA,IACX,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,YAAY,CAAA,CAAE,UAAA;AAAA,IACd,OAAO,UAAA,CAAW,WAAA,EAAa,CAAA,CAAE,MAAA,EAAQ,MAAM,MAAM;AAAA,GACvD,CAAE,CAAA;AAGF,EAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,MAAA,KAAW,WAAA,GAAc,CAAA,GAAI,EAAA;AACjD,EAAA,MAAA,CAAO,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,SAAS,CAAA,CAAE,KAAA,GAAQ,EAAE,KAAA,CAAM,CAAA;AACjD,EAAA,OAAO,MAAA,CAAO,KAAA,CAAM,CAAA,EAAG,CAAC,CAAA;AAC1B;AAEA,SAAS,SAAA,CAAU,SAA4B,SAAA,EAAgD;AAC7F,EAAA,IAAI,SAAA,SAAkB,OAAA,CAAQ,MAAA,CAAO,KAAK,CAAC,CAAA,KAAM,CAAA,CAAE,KAAA,KAAU,SAAS,CAAA;AACtE,EAAA,OAAO,OAAA,CAAQ,OAAO,CAAC,CAAA;AACzB;AAEA,SAAS,UAAA,CAAW,CAAA,EAAiB,CAAA,EAAiB,MAAA,EAAiC;AACrF,EAAA,IAAI,WAAW,WAAA,EAAa;AAC1B,IAAA,IAAI,GAAA,GAAM,CAAA;AACV,IAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,KAAK,CAAA,EAAG;AACpC,MAAA,MAAM,CAAA,GAAI,CAAA,CAAE,CAAC,CAAA,GAAK,EAAE,CAAC,CAAA;AACrB,MAAA,GAAA,IAAO,CAAA,GAAI,CAAA;AAAA,IACb;AACA,IAAA,OAAO,IAAA,CAAK,KAAK,GAAG,CAAA;AAAA,EACtB;AACA,EAAA,IAAI,GAAA,GAAM,CAAA;AACV,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,GAAA,IAAO,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACxD,EAAA,IAAI,MAAA,KAAW,OAAO,OAAO,GAAA;AAE7B,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,IAAI,EAAA,GAAK,CAAA;AACT,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,KAAK,CAAA,EAAG;AACpC,IAAA,EAAA,IAAM,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACjB,IAAA,EAAA,IAAM,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AAAA,EACnB;AACA,EAAA,MAAM,QAAQ,IAAA,CAAK,IAAA,CAAK,EAAE,CAAA,GAAI,IAAA,CAAK,KAAK,EAAE,CAAA;AAC1C,EAAA,OAAO,KAAA,KAAU,CAAA,GAAI,CAAA,GAAI,GAAA,GAAM,KAAA;AACjC;;;AClDA,IAAM,gBAAA,GAAmB,mDAAA;AAElB,SAAS,yBAAyB,IAAA,EAAmD;AAC1F,EAAA,MAAM,QAAQ,IAAA,CAAK,KAAA,IAAS,QAAQ,GAAA,CAAI,QAAA,IAAY,QAAQ,GAAA,CAAI,iBAAA;AAChE,EAAA,IAAI,CAAC,KAAA,EAAO;AACV,IAAA,MAAM,IAAI,MAAM,0EAA0E,CAAA;AAAA,EAC5F;AACA,EAAA,MAAM,OAAA,GAAU,KAAK,OAAA,IAAW,gBAAA;AAChC,EAAA,MAAM,MAAM,CAAA,EAAG,OAAO,IAAI,SAAA,CAAU,IAAA,CAAK,KAAK,CAAC,CAAA,4BAAA,CAAA;AAE/C,EAAA,MAAM,OAAA,GAA4B;AAAA,IAChC,OAAO,IAAA,CAAK,KAAA;AAAA,IACZ,aAAA,EAAe,KAAK,aAAA,IAAiB,MAAA;AAAA,IACrC,MAAA,EAAQ,KAAK,MAAA,IAAU,QAAA;AAAA,IACvB,UAAA,EAAY,IAAA;AAAA,IACZ,MAAM,MAAM,KAAA,EAA2C;AACrD,MAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,QAAA,OAAO,EAAE,OAAA,EAAS,IAAI,SAAA,EAAW,IAAA,CAAK,aAAa,CAAA,EAAE;AAAA,MACvD;AACA,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,SAAA,CAAU,EAAE,MAAA,EAAQ,KAAA,EAAO,OAAA,EAAS,EAAE,cAAA,EAAgB,IAAA,EAAK,EAAG,CAAA;AAChF,MAAA,MAAM,GAAA,GAAM,MAAM,KAAA,CAAM,GAAA,EAAK;AAAA,QAC3B,MAAA,EAAQ,MAAA;AAAA,QACR,OAAA,EAAS;AAAA,UACP,cAAA,EAAgB,kBAAA;AAAA,UAChB,aAAA,EAAe,UAAU,KAAK,CAAA;AAAA,SAChC;AAAA,QACA;AAAA,OACD,CAAA;AACD,MAAA,IAAI,CAAC,IAAI,EAAA,EAAI;AACX,QAAA,MAAM,SAAS,MAAM,GAAA,CAAI,MAAK,CAAE,KAAA,CAAM,MAAM,EAAE,CAAA;AAC9C,QAAA,MAAM,IAAI,KAAA,CAAM,CAAA,iBAAA,EAAoB,GAAA,CAAI,MAAM,CAAA,KAAA,EAAQ,IAAA,CAAK,KAAK,CAAA,EAAA,EAAK,MAAA,IAAU,GAAA,CAAI,UAAU,CAAA,CAAE,CAAA;AAAA,MACjG;AACA,MAAA,MAAM,GAAA,GAAO,MAAM,GAAA,CAAI,IAAA,EAAK;AAC5B,MAAA,MAAM,MAAA,GAAS,eAAA,CAAgB,GAAA,EAAK,KAAA,CAAM,MAAM,CAAA;AAChD,MAAA,MAAM,YAAY,MAAA,CAAO,CAAC,CAAA,EAAG,MAAA,IAAU,KAAK,SAAA,IAAa,CAAA;AACzD,MAAA,MAAM,OAAA,GAAU,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,UAAU,YAAA,CAAa,IAAA,CAAK,CAAC,CAAC,CAAC,CAAA;AACjE,MAAA,OAAO,EAAE,SAAS,SAAA,EAAU;AAAA,IAC9B;AAAA,GACF;AACA,EAAA,IAAI,IAAA,CAAK,SAAA,KAAc,MAAA,EAAW,OAAA,CAAQ,YAAY,IAAA,CAAK,SAAA;AAC3D,EAAA,OAAO,OAAA;AACT;AAWA,SAAS,eAAA,CAAgB,KAAc,aAAA,EAAmC;AACxE,EAAA,IAAI,CAAC,KAAA,CAAM,OAAA,CAAQ,GAAG,CAAA,EAAG;AACvB,IAAA,MAAM,IAAI,MAAM,2CAA2C,CAAA;AAAA,EAC7D;AACA,EAAA,IAAI,GAAA,CAAI,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AAE9B,EAAA,MAAM,KAAA,GAAQ,IAAI,CAAC,CAAA;AACnB,EAAA,IAAI,OAAO,UAAU,QAAA,EAAU;AAC7B,IAAA,IAAI,kBAAkB,CAAA,EAAG;AACvB,MAAA,MAAM,IAAI,KAAA,CAAM,CAAA,yCAAA,EAA4C,aAAa,CAAA,CAAE,CAAA;AAAA,IAC7E;AACA,IAAA,OAAO,CAAC,GAAe,CAAA;AAAA,EACzB;AACA,EAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,KAAM,KAAA,CAAM,MAAA,KAAW,CAAA,IAAK,OAAO,KAAA,CAAM,CAAC,CAAA,KAAM,QAAA,CAAA,EAAW;AAChF,IAAA,OAAO,GAAA;AAAA,EACT;AACA,EAAA,IAAI,KAAA,CAAM,QAAQ,KAAK,CAAA,IAAK,MAAM,OAAA,CAAQ,KAAA,CAAM,CAAC,CAAC,CAAA,EAAG;AAEnD,IAAA,OAAQ,GAAA,CAAqB,IAAI,QAAQ,CAAA;AAAA,EAC3C;AACA,EAAA,MAAM,IAAI,MAAM,+CAA+C,CAAA;AACjE;AAEA,SAAS,SAAS,MAAA,EAA8B;AAC9C,EAAA,IAAI,MAAA,CAAO,MAAA,KAAW,CAAA,EAAG,OAAO,EAAC;AACjC,EAAA,MAAM,GAAA,GAAM,MAAA,CAAO,CAAC,CAAA,CAAG,MAAA;AACvB,EAAA,MAAM,MAAM,IAAI,KAAA,CAAc,GAAG,CAAA,CAAE,KAAK,CAAC,CAAA;AACzC,EAAA,KAAA,MAAW,KAAK,MAAA,EAAQ;AACtB,IAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,GAAA,EAAK,CAAA,IAAK,GAAG,GAAA,CAAI,CAAC,CAAA,IAAM,CAAA,CAAE,CAAC,CAAA;AAAA,EACjD;AACA,EAAA,KAAA,IAAS,CAAA,GAAI,GAAG,CAAA,GAAI,GAAA,EAAK,KAAK,CAAA,EAAG,GAAA,CAAI,CAAC,CAAA,IAAM,MAAA,CAAO,MAAA;AACnD,EAAA,OAAO,GAAA;AACT;AAEA,SAAS,UAAU,CAAA,EAA+B;AAChD,EAAA,IAAI,GAAA,GAAM,CAAA;AACV,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,GAAA,IAAO,CAAA,CAAE,CAAC,CAAA,GAAK,CAAA,CAAE,CAAC,CAAA;AACxD,EAAA,MAAM,IAAA,GAAO,IAAA,CAAK,IAAA,CAAK,GAAG,CAAA;AAC1B,EAAA,IAAI,IAAA,KAAS,GAAG,OAAO,CAAA;AACvB,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,CAAE,MAAA,EAAQ,CAAA,IAAK,CAAA,EAAG,CAAA,CAAE,CAAC,CAAA,GAAI,CAAA,CAAE,CAAC,CAAA,GAAK,IAAA;AACrD,EAAA,OAAO,CAAA;AACT","file":"index.js","sourcesContent":["/**\n * Section-based markdown chunker.\n *\n * Splits a markdown document on ATX headings (`#`, `##`, ...). Each chunk\n * carries the byte offset and length into the original UTF-8 source so a\n * downstream consumer can map a vector hit back to the exact substring\n * without re-tokenising. Pre-heading content becomes a \"preamble\" chunk.\n */\n\nexport type ChunkingMode = 'document' | 'section' | 'paragraph';\n\nexport interface MarkdownChunk {\n  id: string;\n  textOffset: number;\n  textLength: number;\n  text: string;\n}\n\nexport interface ChunkOptions {\n  mode?: ChunkingMode;\n}\n\nconst HEADING = /^(#{1,6})\\s+(.+?)\\s*$/;\n\nexport function chunkMarkdown(markdown: string, opts: ChunkOptions = {}): MarkdownChunk[] {\n  const mode = opts.mode ?? 'section';\n  if (mode === 'document') {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  if (mode === 'paragraph') {\n    return paragraphChunks(markdown);\n  }\n  return sectionChunks(markdown);\n}\n\nfunction sectionChunks(markdown: string): MarkdownChunk[] {\n  const lines = splitWithOffsets(markdown);\n  const sections: MarkdownChunk[] = [];\n  let current: { id: string; start: number; end: number } | null = null;\n  const ids = new Set<string>();\n\n  function flush(end: number): void {\n    if (!current) return;\n    const text = markdown.slice(current.start, end);\n    if (text.trim().length === 0) {\n      current = null;\n      return;\n    }\n    sections.push({ id: current.id, textOffset: current.start, textLength: end - current.start, text });\n    current = null;\n  }\n\n  for (const line of lines) {\n    const match = HEADING.exec(line.text);\n    if (match) {\n      flush(line.offset);\n      const id = uniqueId(slugify(match[2] ?? `section-${sections.length + 1}`), ids);\n      ids.add(id);\n      current = { id, start: line.offset, end: line.offset + line.text.length };\n      continue;\n    }\n    if (current === null) {\n      const id = uniqueId('preamble', ids);\n      ids.add(id);\n      current = { id, start: line.offset, end: line.offset + line.text.length };\n    } else {\n      current.end = line.offset + line.text.length;\n    }\n  }\n  flush(markdown.length);\n\n  if (sections.length === 0) {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  return sections;\n}\n\nfunction paragraphChunks(markdown: string): MarkdownChunk[] {\n  const out: MarkdownChunk[] = [];\n  const ids = new Set<string>();\n  let cursor = 0;\n  let i = 0;\n  while (cursor < markdown.length) {\n    let end = markdown.indexOf('\\n\\n', cursor);\n    if (end === -1) end = markdown.length;\n    const text = markdown.slice(cursor, end);\n    if (text.trim().length > 0) {\n      const id = uniqueId(slugify(text.split('\\n')[0] ?? `p-${i}`), ids);\n      ids.add(id);\n      out.push({ id, textOffset: cursor, textLength: text.length, text });\n      i += 1;\n    }\n    cursor = end + 2;\n  }\n  if (out.length === 0) {\n    return [{ id: 'document', textOffset: 0, textLength: markdown.length, text: markdown }];\n  }\n  return out;\n}\n\nfunction splitWithOffsets(s: string): { text: string; offset: number }[] {\n  const lines: { text: string; offset: number }[] = [];\n  let offset = 0;\n  for (const line of s.split('\\n')) {\n    const withNl = line + (offset + line.length < s.length ? '\\n' : '');\n    lines.push({ text: withNl, offset });\n    offset += withNl.length;\n  }\n  return lines;\n}\n\nfunction slugify(s: string): string {\n  return (\n    s\n      .toLowerCase()\n      .replace(/[^a-z0-9]+/g, '-')\n      .replace(/^-+|-+$/g, '')\n      .slice(0, 64) || 'section'\n  );\n}\n\nfunction uniqueId(base: string, taken: Set<string>): string {\n  if (!taken.has(base)) return base;\n  let n = 2;\n  while (taken.has(`${base}-${n}`)) n += 1;\n  return `${base}-${n}`;\n}\n","/**\n * transformers.js backend (browser + Node + Bun via @huggingface/transformers).\n *\n * Loads the model lazily and reuses the pipeline across calls. Models are\n * cached on disk by transformers.js itself (HF cache layout).\n */\n\nimport { pipeline, env } from '@huggingface/transformers';\nimport type { EmbeddingBackend, EmbeddingBackendOptions, EmbeddingMatrix } from './types.js';\n\ninterface TransformersBackendOptions extends EmbeddingBackendOptions {\n  /** Override Xenova quantisation. Defaults to fp32 for closest fidelity. */\n  dtype?: 'fp32' | 'fp16' | 'q8' | 'q4';\n  /** Force a backend device (e.g. 'cpu', 'gpu', 'wasm'). Defaults to auto. */\n  device?: 'cpu' | 'gpu' | 'wasm' | 'webgpu';\n  /** Allow remote model downloads. true by default. */\n  allowRemoteModels?: boolean;\n}\n\nexport function createTransformersBackend(opts: TransformersBackendOptions): EmbeddingBackend {\n  if (opts.allowRemoteModels === false) {\n    env.allowRemoteModels = false;\n  }\n\n  let pipelinePromise: Promise<unknown> | null = null;\n  let resolvedDimension: number | null = null;\n\n  function getPipeline(): Promise<unknown> {\n    if (!pipelinePromise) {\n      const modelOpts: Record<string, unknown> = { dtype: opts.dtype ?? 'fp32' };\n      if (opts.device !== undefined) modelOpts.device = opts.device;\n      pipelinePromise = pipeline('feature-extraction', opts.model, modelOpts as never);\n    }\n    return pipelinePromise;\n  }\n\n  const backend: EmbeddingBackend = {\n    model: opts.model,\n    modelRevision: opts.modelRevision ?? 'main',\n    metric: opts.metric ?? 'cosine',\n    normalized: true,\n    async embed(texts: string[]): Promise<EmbeddingMatrix> {\n      if (texts.length === 0) {\n        return { vectors: [], dimension: opts.dimension ?? 0 };\n      }\n      const pipe = (await getPipeline()) as (\n        texts: string[],\n        opts: { pooling: 'mean'; normalize: boolean },\n      ) => Promise<{ data: Float32Array; dims: number[] }>;\n      const tensor = await pipe(texts, { pooling: 'mean', normalize: true });\n      const data = tensor.data;\n      const dimension = tensor.dims[tensor.dims.length - 1] as number;\n      if (resolvedDimension !== null && resolvedDimension !== dimension) {\n        throw new Error(`Model emitted inconsistent dimension: had ${resolvedDimension}, now ${dimension}`);\n      }\n      resolvedDimension = dimension;\n      const vectors: Float32Array[] = [];\n      for (let i = 0; i < texts.length; i += 1) {\n        vectors.push(new Float32Array(data.buffer, data.byteOffset + i * dimension * 4, dimension).slice());\n      }\n      return { vectors, dimension };\n    },\n  };\n  if (opts.dimension !== undefined) backend.dimension = opts.dimension;\n  return backend;\n}\n","import type { EmbeddingMetric } from '@cvfile/sdk';\n\nexport interface EmbeddingMatrix {\n  vectors: Float32Array[];\n  dimension: number;\n}\n\nexport interface EmbeddingBackend {\n  model: string;\n  modelRevision: string;\n  /** Optional pre-declared dimension; the backend may override after first call. */\n  dimension?: number;\n  metric: EmbeddingMetric;\n  normalized: boolean;\n  embed(texts: string[]): Promise<EmbeddingMatrix>;\n}\n\nexport interface EmbeddingBackendOptions {\n  model: string;\n  modelRevision?: string;\n  /** Optional pre-known dimension (e.g. 1024 for bge-m3). */\n  dimension?: number;\n  metric?: EmbeddingMetric;\n}\n\n/** Recommended default per spec §5: BAAI BGE-M3, MIT, multilingual, 1024-dim. */\nexport const DEFAULT_MODEL = 'Xenova/bge-m3';\nexport const DEFAULT_MODEL_DIMENSION = 1024;\n","/**\n * High-level embed() API: markdown in, EmbeddingsPayload out, ready to drop\n * into pack({ embeddings: ... }).\n */\n\nimport type { EmbeddingChunk, EmbeddingSpace, EmbeddingsPayload } from '@cvfile/sdk';\nimport { chunkMarkdown, type ChunkingMode } from './chunk.js';\nimport { createTransformersBackend } from './transformers-backend.js';\nimport { DEFAULT_MODEL, type EmbeddingBackend } from './types.js';\n\nexport interface EmbedOptions {\n  /** HF model id; defaults to BGE-M3 (Xenova/bge-m3). */\n  model?: string;\n  /** Pinned model revision; recorded in the payload for reproducibility. */\n  modelRevision?: string;\n  /** Chunking strategy; default 'section'. */\n  chunking?: ChunkingMode;\n  /** Bring-your-own backend (e.g. an OpenAI/Voyage adapter). */\n  backend?: EmbeddingBackend;\n}\n\nexport async function embed(markdown: string, opts: EmbedOptions = {}): Promise<EmbeddingsPayload> {\n  const chunks = chunkMarkdown(markdown, { mode: opts.chunking ?? 'section' });\n  const backend =\n    opts.backend ??\n    createTransformersBackend(\n      opts.modelRevision !== undefined\n        ? { model: opts.model ?? DEFAULT_MODEL, modelRevision: opts.modelRevision }\n        : { model: opts.model ?? DEFAULT_MODEL },\n    );\n\n  const matrix = await backend.embed(chunks.map((c) => c.text));\n  if (matrix.vectors.length !== chunks.length) {\n    throw new Error(`Backend returned ${matrix.vectors.length} vectors for ${chunks.length} chunks`);\n  }\n\n  const embeddingChunks: EmbeddingChunk[] = chunks.map((c, i) => ({\n    id: c.id,\n    textOffset: c.textOffset,\n    textLength: c.textLength,\n    vector: matrix.vectors[i]!,\n  }));\n\n  const space: EmbeddingSpace = {\n    model: backend.model,\n    modelRevision: backend.modelRevision,\n    dimension: matrix.dimension,\n    metric: backend.metric,\n    normalized: backend.normalized,\n    chunking: opts.chunking ?? 'section',\n    chunks: embeddingChunks,\n  };\n\n  return { formatVersion: 1, spaces: [space] };\n}\n","/**\n * Pure similarity search over an EmbeddingsPayload. Vector input is the\n * caller's responsibility (encode the query with the same model that was\n * used to populate the space, ideally pulled from space.model).\n */\n\nimport type { EmbeddingMetric, EmbeddingSpace, EmbeddingsPayload } from '@cvfile/sdk';\n\nexport interface SearchHit {\n  spaceModel: string;\n  chunkId: string;\n  textOffset: number;\n  textLength: number;\n  score: number;\n}\n\nexport interface SearchOptions {\n  /** Restrict to a specific embedding space; default = first space. */\n  model?: string;\n  /** Top-k results; default 5. */\n  k?: number;\n}\n\nexport function searchSemantic(\n  payload: EmbeddingsPayload,\n  queryVector: Float32Array,\n  opts: SearchOptions = {},\n): SearchHit[] {\n  const space = pickSpace(payload, opts.model);\n  if (!space) throw new Error('No matching embedding space found');\n  if (queryVector.length !== space.dimension) {\n    throw new Error(`Query vector dimension ${queryVector.length} does not match space ${space.model} (${space.dimension})`);\n  }\n\n  const k = opts.k ?? 5;\n  const scored: SearchHit[] = space.chunks.map((c) => ({\n    spaceModel: space.model,\n    chunkId: c.id,\n    textOffset: c.textOffset,\n    textLength: c.textLength,\n    score: similarity(queryVector, c.vector, space.metric),\n  }));\n\n  // Higher score = better for cosine/dot; lower for euclidean.\n  const order = space.metric === 'euclidean' ? 1 : -1;\n  scored.sort((a, b) => order * (a.score - b.score));\n  return scored.slice(0, k);\n}\n\nfunction pickSpace(payload: EmbeddingsPayload, modelHint?: string): EmbeddingSpace | undefined {\n  if (modelHint) return payload.spaces.find((s) => s.model === modelHint);\n  return payload.spaces[0];\n}\n\nfunction similarity(a: Float32Array, b: Float32Array, metric: EmbeddingMetric): number {\n  if (metric === 'euclidean') {\n    let sum = 0;\n    for (let i = 0; i < a.length; i += 1) {\n      const d = a[i]! - b[i]!;\n      sum += d * d;\n    }\n    return Math.sqrt(sum);\n  }\n  let dot = 0;\n  for (let i = 0; i < a.length; i += 1) dot += a[i]! * b[i]!;\n  if (metric === 'dot') return dot;\n  // cosine: assume normalized vectors when produced by our backend\n  let na = 0;\n  let nb = 0;\n  for (let i = 0; i < a.length; i += 1) {\n    na += a[i]! * a[i]!;\n    nb += b[i]! * b[i]!;\n  }\n  const denom = Math.sqrt(na) * Math.sqrt(nb);\n  return denom === 0 ? 0 : dot / denom;\n}\n","/**\n * Hugging Face Inference API backend.\n *\n * Calls https://router.huggingface.co/hf-inference/models/<model>/pipeline/feature-extraction\n * with HF_TOKEN. For sentence-transformers models (BGE-M3, MiniLM, etc.) the\n * response is already mean-pooled per input — one vector per text. We\n * normalise client-side so cosine math is consistent across backends.\n */\n\nimport type { EmbeddingBackend, EmbeddingMatrix } from './types.js';\nimport type { EmbeddingMetric } from '@cvfile/sdk';\n\nexport interface HuggingFaceBackendOptions {\n  model: string;\n  /** HF token. Defaults to `process.env.HF_TOKEN`. */\n  token?: string;\n  /** Pinned revision; recorded in the payload. Default 'main'. */\n  modelRevision?: string;\n  /** Pre-known dimension. Optional; inferred from first response otherwise. */\n  dimension?: number;\n  metric?: EmbeddingMetric;\n  /** Override base URL (e.g. for self-hosted TEI). */\n  baseUrl?: string;\n}\n\nconst DEFAULT_BASE_URL = 'https://router.huggingface.co/hf-inference/models';\n\nexport function createHuggingFaceBackend(opts: HuggingFaceBackendOptions): EmbeddingBackend {\n  const token = opts.token ?? process.env.HF_TOKEN ?? process.env.HUGGINGFACE_TOKEN;\n  if (!token) {\n    throw new Error('HF_TOKEN (or HUGGINGFACE_TOKEN) is required for the Hugging Face backend');\n  }\n  const baseUrl = opts.baseUrl ?? DEFAULT_BASE_URL;\n  const url = `${baseUrl}/${encodeURI(opts.model)}/pipeline/feature-extraction`;\n\n  const backend: EmbeddingBackend = {\n    model: opts.model,\n    modelRevision: opts.modelRevision ?? 'main',\n    metric: opts.metric ?? 'cosine',\n    normalized: true,\n    async embed(texts: string[]): Promise<EmbeddingMatrix> {\n      if (texts.length === 0) {\n        return { vectors: [], dimension: opts.dimension ?? 0 };\n      }\n      const body = JSON.stringify({ inputs: texts, options: { wait_for_model: true } });\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          authorization: `Bearer ${token}`,\n        },\n        body,\n      });\n      if (!res.ok) {\n        const detail = await res.text().catch(() => '');\n        throw new Error(`HF Inference API ${res.status} for ${opts.model}: ${detail || res.statusText}`);\n      }\n      const raw = (await res.json()) as unknown;\n      const matrix = parseHfResponse(raw, texts.length);\n      const dimension = matrix[0]?.length ?? opts.dimension ?? 0;\n      const vectors = matrix.map((v) => normalize(Float32Array.from(v)));\n      return { vectors, dimension };\n    },\n  };\n  if (opts.dimension !== undefined) backend.dimension = opts.dimension;\n  return backend;\n}\n\n/**\n * Coerce the variety of shapes the HF Inference API returns into a flat\n * `number[][]`: one mean-pooled vector per input.\n *\n * Observed shapes:\n *   - sentence-transformers (BGE-M3, MiniLM): `[[...vec], [...vec]]`\n *   - feature-extraction without pooling: `[[[...token0], [...token1], ...], ...]`\n *   - single-input convenience form: `[...vec]`\n */\nfunction parseHfResponse(raw: unknown, expectedCount: number): number[][] {\n  if (!Array.isArray(raw)) {\n    throw new Error('HF Inference API: expected array response');\n  }\n  if (raw.length === 0) return [];\n\n  const first = raw[0];\n  if (typeof first === 'number') {\n    if (expectedCount !== 1) {\n      throw new Error(`HF Inference API: got 1 vector, expected ${expectedCount}`);\n    }\n    return [raw as number[]];\n  }\n  if (Array.isArray(first) && (first.length === 0 || typeof first[0] === 'number')) {\n    return raw as number[][];\n  }\n  if (Array.isArray(first) && Array.isArray(first[0])) {\n    // token-level embeddings: mean-pool per input\n    return (raw as number[][][]).map(meanPool);\n  }\n  throw new Error('HF Inference API: unrecognised response shape');\n}\n\nfunction meanPool(tokens: number[][]): number[] {\n  if (tokens.length === 0) return [];\n  const dim = tokens[0]!.length;\n  const out = new Array<number>(dim).fill(0);\n  for (const t of tokens) {\n    for (let i = 0; i < dim; i += 1) out[i]! += t[i]!;\n  }\n  for (let i = 0; i < dim; i += 1) out[i]! /= tokens.length;\n  return out;\n}\n\nfunction normalize(v: Float32Array): Float32Array {\n  let sum = 0;\n  for (let i = 0; i < v.length; i += 1) sum += v[i]! * v[i]!;\n  const norm = Math.sqrt(sum);\n  if (norm === 0) return v;\n  for (let i = 0; i < v.length; i += 1) v[i] = v[i]! / norm;\n  return v;\n}\n"]}

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@cvfile/embed",
+  "version": "0.1.0",
+  "description": "Open-source embedding generation for the .cv format. BGE-M3 by default; pluggable.",
+  "license": "Apache-2.0",
+  "homepage": "https://cvfile.org",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/cvfile/cv",
+    "directory": "packages/embed-js"
+  },
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "sideEffects": false,
+  "dependencies": {
+    "@huggingface/transformers": "^3.0.0",
+    "@cvfile/sdk": "0.1.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "tsup": "^8.3.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.1.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist .turbo"
+  }
+}