@lacspace/condense 1.0.0

package/LICENSE

@@ -0,0 +1,51 @@
+Lacspace Free Licence
+Version 1.0, August 2026
+
+Copyright (c) 2026 Lacspace
+
+PREAMBLE
+
+This software is published by Lacspace under the Lacspace Free Licence — a free,
+permissive licence that lets you use this software for any purpose, including in
+commercial products and services, at no cost. It grants the same freedoms as
+common permissive open-source licences; the only condition is that this notice
+travels with the software. The canonical, always-current text of this licence is
+maintained at https://lacspace.com/licenses/lacspace-free-1.0
+
+GRANT OF RIGHTS
+
+Permission is hereby granted, free of charge, to any person or organisation
+obtaining a copy of this software and its associated documentation and data files
+(the "Software"), to deal in the Software without restriction, including without
+limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the conditions below. These rights are
+granted for any purpose, personal or commercial, and are perpetual, worldwide,
+non-exclusive, and royalty-free.
+
+CONDITIONS
+
+The above copyright notice, this permission notice, and the name of this licence
+("Lacspace Free Licence") shall be included in all copies or substantial portions
+of the Software.
+
+TRADEMARKS
+
+This licence does not grant permission to use the trade names, trademarks, service
+marks, logos, or product names of Lacspace, except as required to reproduce the
+notice above or to describe the origin of the Software in a truthful manner.
+
+DISCLAIMER OF WARRANTY AND LIMITATION OF LIABILITY
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER IN
+AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF, OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---
+
+The Lacspace Free Licence is a source-available, permissive licence and is not (as
+of this version) an OSI-approved licence. In substance it grants the same freedoms
+as the MIT Licence. Learn more at https://lacspace.com/licenses

package/README.md

@@ -0,0 +1,86 @@
+# @lacspace/condense
+
+**Feed the LLM a fraction of the words.** Turn several articles about the same story into one short, deduplicated, token-budgeted digest that keeps the numbers, quotes and named entities — so a model only has to rewrite what matters instead of re-reading six full sources. Purely extractive, deterministic, and zero external dependencies.
+
+Built for newsrooms, RAG context assembly, and anywhere you're paying per token to summarise overlapping documents.
+
+```bash
+npm i @lacspace/condense
+```
+
+## Use
+
+```ts
+import { condense } from "@lacspace/condense";
+
+const digest = condense(
+  [
+    { text: kathmanduPostArticle, label: "Kathmandu Post" },
+    { text: himalayanTimesArticle, label: "Himalayan Times" },
+    { text: onlineKhabarArticle, label: "Online Khabar" },
+  ],
+  { tokenBudget: 1500, gazetteer: ["Nepal Rastra Bank", "नेपाल राष्ट्र बैंक"] },
+);
+
+digest.text;
+// [S1 Kathmandu Post]
+// Nepal's central bank cut the policy rate to 5.5 percent on Sunday. The governor said "inflation is easing".
+//
+// [S2 Himalayan Times]
+// Analysts welcomed the decision.
+//
+// [S3 Online Khabar]
+// Traders expect loans to get cheaper.
+
+digest.tokens;      // ≤ tokenBudget
+digest.droppedDup;  // near-duplicate sentences removed across sources
+```
+
+Then hand `digest.text` to your model. On a typical 6-source cluster this is ~50% of the input tokens of feeding the trimmed sources directly, with the facts preserved.
+
+## What it does
+
+- **Ranks sentences** by BM25 centrality across the whole cluster (via `@lacspace/rerank`).
+- **Always keeps** each source's lede, any sentence with a number/currency/percentage, any quoted sentence, and any sentence naming a gazetteer entity — these are tagged in `reasons`.
+- **Drops near-duplicates** (3-gram Jaccard) so five outlets saying the same figure appear once.
+- **Groups per source** under a header line in the original source order, in each source's original reading order — so your prompt can still credit "according to S2".
+- **Respects a token budget** (`@lacspace/tokenizer`) and a `maxSentencesPerSource` cap so one long article can't crowd out the rest.
+- **Devanagari-aware**: splits on `।` and `॥`, detects `०-९` digits and `रु`/`नेरु` amounts, never splits a quote.
+- **Deterministic**: identical input always yields identical output, so results are cacheable.
+
+## API
+
+```ts
+condense(sources: CondenseSource[], options?: CondenseOptions): CondenseResult
+```
+
+```ts
+interface CondenseSource { text: string; label?: string; url?: string; publishedAt?: string | Date; }
+
+interface CondenseOptions {
+  tokenBudget?: number;            // default 1500
+  maxSentencesPerSource?: number;  // default 8
+  dedupeThreshold?: number;        // 3-gram Jaccard ≥ this → drop as duplicate. default 0.8
+  keepLede?: boolean;              // default true
+  gazetteer?: string[];            // entities to always keep + boost (pass en and ne forms)
+  model?: string;                  // token-counter model hint
+  countTokens?: (text: string) => number;  // override token counting
+  header?: (s: { idx: number; label: string; url?: string }) => string;
+}
+
+interface CondensedSentence { text: string; sourceIdx: number; start: number; end: number; score: number; reasons: string[]; }
+interface CondenseResult {
+  text: string;                    // grouped digest
+  sources: { idx; label; url?; sentences: CondensedSentence[] }[];
+  sentences: CondensedSentence[];  // flat, in output order
+  tokens: number; droppedDup: number; totalSentences: number;
+}
+```
+
+Every kept sentence carries `sourceIdx` and the `{start, end}` char offsets in that source's original text — so an editor view or a plagiarism check can trace each line back to where it came from. `condense` never rewrites text; publishing generated prose is the model's job.
+
+Also exported: `splitSentences(text)` — the Devanagari-aware sentence splitter with offsets, useful on its own.
+
+## Licence
+
+[Lacspace Free Licence v1.0](https://developer.lacspace.com/licenses/lacspace-free-1.0) — free for personal and commercial use.

package/dist/index.cjs

@@ -0,0 +1,255 @@
+'use strict';
+
+var rerank = require('@lacspace/rerank');
+var tokenizer = require('@lacspace/tokenizer');
+
+// src/index.ts
+
+// src/features.ts
+var CURRENCY = /(?:रु|नेरु|रुपैयाँ|₹|\$|€|£|Rs\.?|USD|NPR|INR)/i;
+var NUMBERISH = /[0-9०-९][0-9०-९.,%]*/;
+var PERCENT = /[%]|प्रतिशत/;
+var QUOTED = /"[^"]{3,}"|“[^”]{3,}”|‘[^’]{3,}’/;
+function hasNumber(text) {
+  return NUMBERISH.test(text) || CURRENCY.test(text) || PERCENT.test(text);
+}
+function hasQuote(text) {
+  return QUOTED.test(text);
+}
+function hasEntity(text, gaz) {
+  if (!gaz || gaz.size === 0) return false;
+  const lower = text.toLowerCase();
+  for (const term of gaz) {
+    if (!term) continue;
+    if (/[^\u0000-\u007F]/.test(term)) {
+      if (text.includes(term)) return true;
+    } else if (new RegExp(`(?:^|[^a-z])${escapeRe(term.toLowerCase())}(?:[^a-z]|$)`).test(lower)) {
+      return true;
+    }
+  }
+  return false;
+}
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function shingles(text, size = 3) {
+  const words = text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter(Boolean);
+  const out = /* @__PURE__ */ new Set();
+  if (words.length < size) {
+    if (words.length) out.add(words.join(" "));
+    return out;
+  }
+  for (let i = 0; i + size <= words.length; i++) out.add(words.slice(i, i + size).join(" "));
+  return out;
+}
+function jaccard(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let inter = 0;
+  const [small, large] = a.size <= b.size ? [a, b] : [b, a];
+  for (const t of small) if (large.has(t)) inter++;
+  return inter / (a.size + b.size - inter);
+}
+
+// src/sentences.ts
+var TERMINATORS = /* @__PURE__ */ new Set([".", "!", "?", "\u0964", "\u0965", "\u2026"]);
+var OPEN_QUOTES = /* @__PURE__ */ new Set(["\u201C", "\u2018", "\xAB"]);
+var CLOSE_QUOTES = /* @__PURE__ */ new Set(["\u201D", "\u2019", "\xBB"]);
+var ABBREV = /* @__PURE__ */ new Set([
+  "mr",
+  "mrs",
+  "ms",
+  "dr",
+  "prof",
+  "sr",
+  "jr",
+  "st",
+  "vs",
+  "etc",
+  "inc",
+  "ltd",
+  "co",
+  "corp",
+  "govt",
+  "gen",
+  "rep",
+  "sen",
+  "gov",
+  "no",
+  "vol",
+  "fig",
+  "al",
+  "rs",
+  "u.s",
+  "u.k",
+  "e.g",
+  "i.e",
+  "a.m",
+  "p.m"
+]);
+function isDigit(ch) {
+  return ch >= "0" && ch <= "9" || ch >= "\u0966" && ch <= "\u096F";
+}
+function splitSentences(text) {
+  const out = [];
+  if (!text) return out;
+  let quoteDepth = 0;
+  let doubleOpen = false;
+  let start = 0;
+  const n = text.length;
+  const push = (from, to) => {
+    const raw = text.slice(from, to);
+    const trimmedStart = raw.length - raw.trimStart().length;
+    const trimmedEnd = raw.length - raw.trimEnd().length;
+    const s = from + trimmedStart;
+    const e = to - trimmedEnd;
+    if (e > s) out.push({ text: text.slice(s, e), start: s, end: e });
+  };
+  for (let i = 0; i < n; i++) {
+    const ch = text[i];
+    if (ch === '"') {
+      doubleOpen = !doubleOpen;
+      continue;
+    }
+    if (OPEN_QUOTES.has(ch)) {
+      quoteDepth++;
+      continue;
+    }
+    if (CLOSE_QUOTES.has(ch)) {
+      if (quoteDepth > 0) quoteDepth--;
+      continue;
+    }
+    if (!TERMINATORS.has(ch)) continue;
+    if (quoteDepth > 0 || doubleOpen) continue;
+    if (ch === ".") {
+      const prev = text[i - 1];
+      const next = text[i + 1];
+      if (prev && next && isDigit(prev) && isDigit(next)) continue;
+      let j = i - 1;
+      while (j >= 0 && /[A-Za-z.]/.test(text[j])) j--;
+      const word = text.slice(j + 1, i).toLowerCase();
+      if (ABBREV.has(word)) continue;
+    }
+    let k = i + 1;
+    while (k < n && (TERMINATORS.has(text[k]) || CLOSE_QUOTES.has(text[k]) || text[k] === '"' || text[k] === ")" || text[k] === "]")) {
+      if (text[k] === '"') doubleOpen = !doubleOpen;
+      k++;
+    }
+    if (k >= n || /\s/.test(text[k])) {
+      push(start, k);
+      start = k;
+      i = k - 1;
+    }
+  }
+  if (start < n) push(start, n);
+  return out;
+}
+
+// src/index.ts
+function condense(sources, options = {}) {
+  const tokenBudget = options.tokenBudget ?? 1500;
+  const maxPer = options.maxSentencesPerSource ?? 8;
+  const dedupe = options.dedupeThreshold ?? 0.8;
+  const keepLede = options.keepLede ?? true;
+  const count = options.countTokens ?? ((t) => tokenizer.countTokens(t, options.model));
+  const gaz = options.gazetteer && options.gazetteer.length ? new Set(options.gazetteer) : null;
+  const header = options.header ?? ((s) => `[S${s.idx + 1} ${s.label}]`);
+  const labels = sources.map((s, i) => s.label?.trim() || `S${i + 1}`);
+  const cands = [];
+  let order = 0;
+  sources.forEach((src, sIdx) => {
+    const sents = splitSentences(src.text ?? "");
+    sents.forEach((sen, localIdx) => {
+      cands.push({
+        text: sen.text,
+        sourceIdx: sIdx,
+        start: sen.start,
+        end: sen.end,
+        order: order++,
+        score: 0,
+        lede: localIdx === 0,
+        number: hasNumber(sen.text),
+        quote: hasQuote(sen.text),
+        entity: hasEntity(sen.text, gaz),
+        shingle: shingles(sen.text),
+        tokens: count(sen.text)
+      });
+    });
+  });
+  const totalSentences = cands.length;
+  if (totalSentences === 0) {
+    return { text: "", sources: [], sentences: [], tokens: 0, droppedDup: 0, totalSentences: 0 };
+  }
+  const clusterQuery = cands.map((c) => c.text).join(" ");
+  const scored = rerank.bm25(
+    clusterQuery,
+    cands.map((c) => ({ id: String(c.order), text: c.text }))
+  );
+  const scoreById = new Map(scored.map((s) => [s.id, s.rerankScore]));
+  for (const c of cands) c.score = scoreById.get(String(c.order)) ?? 0;
+  const priority = (c) => (keepLede && c.lede ? 4 : 0) + (c.quote ? 3 : 0) + (c.number ? 2 : 0) + (c.entity ? 1 : 0);
+  const ranked = [...cands].sort((a, b) => {
+    const pa = priority(a);
+    const pb = priority(b);
+    if (pa !== pb) return pb - pa;
+    if (b.score !== a.score) return b.score - a.score;
+    return a.order - b.order;
+  });
+  const kept = [];
+  const keptShingles = [];
+  const perSource = /* @__PURE__ */ new Map();
+  let used = 0;
+  let droppedDup = 0;
+  for (const c of ranked) {
+    let dup = false;
+    for (const ks of keptShingles) {
+      if (jaccard(c.shingle, ks) >= dedupe) {
+        dup = true;
+        break;
+      }
+    }
+    if (dup) {
+      droppedDup++;
+      continue;
+    }
+    if ((perSource.get(c.sourceIdx) ?? 0) >= maxPer) continue;
+    if (used + c.tokens > tokenBudget && kept.length > 0) continue;
+    kept.push(c);
+    keptShingles.push(c.shingle);
+    perSource.set(c.sourceIdx, (perSource.get(c.sourceIdx) ?? 0) + 1);
+    used += c.tokens;
+  }
+  const bySource = /* @__PURE__ */ new Map();
+  for (const c of kept) {
+    const arr = bySource.get(c.sourceIdx) ?? [];
+    arr.push(c);
+    bySource.set(c.sourceIdx, arr);
+  }
+  const toOut = (c) => {
+    const reasons = [];
+    if (keepLede && c.lede) reasons.push("lede");
+    if (c.quote) reasons.push("quote");
+    if (c.number) reasons.push("number");
+    if (c.entity) reasons.push("entity");
+    if (reasons.length === 0) reasons.push("rank");
+    return { text: c.text, sourceIdx: c.sourceIdx, start: c.start, end: c.end, score: c.score, reasons };
+  };
+  const outSources = [];
+  const flat = [];
+  const parts = [];
+  sources.forEach((src, sIdx) => {
+    const list = (bySource.get(sIdx) ?? []).sort((a, b) => a.start - b.start);
+    if (list.length === 0) return;
+    const outSents = list.map(toOut);
+    outSources.push({ idx: sIdx, label: labels[sIdx], url: src.url, sentences: outSents });
+    flat.push(...outSents);
+    parts.push(`${header({ idx: sIdx, label: labels[sIdx], url: src.url })}
+${outSents.map((s) => s.text).join(" ")}`);
+  });
+  const text = parts.join("\n\n");
+  return { text, sources: outSources, sentences: flat, tokens: count(text), droppedDup, totalSentences };
+}
+
+exports.condense = condense;
+exports.splitSentences = splitSentences;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map

package/dist/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/features.ts","../src/sentences.ts","../src/index.ts"],"names":["countTokens","bm25"],"mappings":";;;;;;;;AAEA,IAAM,QAAA,GAAW,iDAAA;AAEjB,IAAM,SAAA,GAAY,sBAAA;AAClB,IAAM,OAAA,GAAU,aAAA;AAEhB,IAAM,MAAA,GAAS,kCAAA;AAER,SAAS,UAAU,IAAA,EAAuB;AAC/C,EAAA,OAAO,SAAA,CAAU,IAAA,CAAK,IAAI,CAAA,IAAK,QAAA,CAAS,KAAK,IAAI,CAAA,IAAK,OAAA,CAAQ,IAAA,CAAK,IAAI,CAAA;AACzE;AAEO,SAAS,SAAS,IAAA,EAAuB;AAC9C,EAAA,OAAO,MAAA,CAAO,KAAK,IAAI,CAAA;AACzB;AAGO,SAAS,SAAA,CAAU,MAAc,GAAA,EAAkC;AACxE,EAAA,IAAI,CAAC,GAAA,IAAO,GAAA,CAAI,IAAA,KAAS,GAAG,OAAO,KAAA;AACnC,EAAA,MAAM,KAAA,GAAQ,KAAK,WAAA,EAAY;AAC/B,EAAA,KAAA,MAAW,QAAQ,GAAA,EAAK;AACtB,IAAA,IAAI,CAAC,IAAA,EAAM;AAEX,IAAA,IAAI,kBAAA,CAAmB,IAAA,CAAK,IAAI,CAAA,EAAG;AACjC,MAAA,IAAI,IAAA,CAAK,QAAA,CAAS,IAAI,CAAA,EAAG,OAAO,IAAA;AAAA,IAClC,CAAA,MAAA,IAAW,IAAI,MAAA,CAAO,CAAA,YAAA,EAAe,QAAA,CAAS,IAAA,CAAK,WAAA,EAAa,CAAC,CAAA,YAAA,CAAc,CAAA,CAAE,IAAA,CAAK,KAAK,CAAA,EAAG;AAC5F,MAAA,OAAO,IAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,KAAA;AACT;AAEA,SAAS,SAAS,CAAA,EAAmB;AACnC,EAAA,OAAO,CAAA,CAAE,OAAA,CAAQ,qBAAA,EAAuB,MAAM,CAAA;AAChD;AAGO,SAAS,QAAA,CAAS,IAAA,EAAc,IAAA,GAAO,CAAA,EAAgB;AAC5D,EAAA,MAAM,KAAA,GAAQ,IAAA,CACX,WAAA,EAAY,CACZ,OAAA,CAAQ,mBAAA,EAAqB,GAAG,CAAA,CAChC,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAO,OAAO,CAAA;AACjB,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAC5B,EAAA,IAAI,KAAA,CAAM,SAAS,IAAA,EAAM;AACvB,IAAA,IAAI,MAAM,MAAA,EAAQ,GAAA,CAAI,IAAI,KAAA,CAAM,IAAA,CAAK,GAAG,CAAC,CAAA;AACzC,IAAA,OAAO,GAAA;AAAA,EACT;AACA,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,IAAA,IAAQ,KAAA,CAAM,QAAQ,CAAA,EAAA,EAAK,GAAA,CAAI,GAAA,CAAI,KAAA,CAAM,MAAM,CAAA,EAAG,CAAA,GAAI,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAC,CAAA;AACzF,EAAA,OAAO,GAAA;AACT;AAEO,SAAS,OAAA,CAAQ,GAAgB,CAAA,EAAwB;AAC9D,EAAA,IAAI,EAAE,IAAA,KAAS,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,GAAG,OAAO,CAAA;AACzC,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,MAAM,CAAC,KAAA,EAAO,KAAK,CAAA,GAAI,EAAE,IAAA,IAAQ,CAAA,CAAE,IAAA,GAAO,CAAC,CAAA,EAAG,CAAC,CAAA,GAAI,CAAC,GAAG,CAAC,CAAA;AACxD,EAAA,KAAA,MAAW,KAAK,KAAA,EAAO,IAAI,KAAA,CAAM,GAAA,CAAI,CAAC,CAAA,EAAG,KAAA,EAAA;AACzC,EAAA,OAAO,KAAA,IAAS,CAAA,CAAE,IAAA,GAAO,CAAA,CAAE,IAAA,GAAO,KAAA,CAAA;AACpC;;;ACjDA,IAAM,WAAA,mBAAc,IAAI,GAAA,CAAI,CAAC,GAAA,EAAK,KAAK,GAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAG,CAAC,CAAA;AAE1D,IAAM,8BAAc,IAAI,GAAA,CAAI,CAAC,QAAA,EAAK,QAAA,EAAK,MAAG,CAAC,CAAA;AAC3C,IAAM,+BAAe,IAAI,GAAA,CAAI,CAAC,QAAA,EAAK,QAAA,EAAK,MAAG,CAAC,CAAA;AAG5C,IAAM,MAAA,uBAAa,GAAA,CAAI;AAAA,EACrB,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,MAAA;AAAA,EAAQ,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EACvE,IAAA;AAAA,EAAM,MAAA;AAAA,EAAQ,MAAA;AAAA,EAAQ,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,IAAA;AAAA,EACtE,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO;AAC3C,CAAC,CAAA;AAED,SAAS,QAAQ,EAAA,EAAqB;AACpC,EAAA,OAAQ,MAAM,GAAA,IAAO,EAAA,IAAM,GAAA,IAAS,EAAA,IAAM,YAAO,EAAA,IAAM,QAAA;AACzD;AAQO,SAAS,eAAe,IAAA,EAA0B;AACvD,EAAA,MAAM,MAAkB,EAAC;AACzB,EAAA,IAAI,CAAC,MAAM,OAAO,GAAA;AAClB,EAAA,IAAI,UAAA,GAAa,CAAA;AACjB,EAAA,IAAI,UAAA,GAAa,KAAA;AACjB,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AAEf,EAAA,MAAM,IAAA,GAAO,CAAC,IAAA,EAAc,EAAA,KAAe;AACzC,IAAA,MAAM,GAAA,GAAM,IAAA,CAAK,KAAA,CAAM,IAAA,EAAM,EAAE,CAAA;AAC/B,IAAA,MAAM,YAAA,GAAe,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,WAAU,CAAE,MAAA;AAClD,IAAA,MAAM,UAAA,GAAa,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,SAAQ,CAAE,MAAA;AAC9C,IAAA,MAAM,IAAI,IAAA,GAAO,YAAA;AACjB,IAAA,MAAM,IAAI,EAAA,GAAK,UAAA;AACf,IAAA,IAAI,CAAA,GAAI,CAAA,EAAG,GAAA,CAAI,IAAA,CAAK,EAAE,IAAA,EAAM,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,CAAC,CAAA,EAAG,KAAA,EAAO,CAAA,EAAG,GAAA,EAAK,GAAG,CAAA;AAAA,EAClE,CAAA;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,EAAG,CAAA,EAAA,EAAK;AAC1B,IAAA,MAAM,EAAA,GAAK,KAAK,CAAC,CAAA;AACjB,IAAA,IAAI,OAAO,GAAA,EAAK;AACd,MAAA,UAAA,GAAa,CAAC,UAAA;AACd,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAA,CAAY,GAAA,CAAI,EAAE,CAAA,EAAG;AACvB,MAAA,UAAA,EAAA;AACA,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,GAAA,CAAI,EAAE,CAAA,EAAG;AACxB,MAAA,IAAI,aAAa,CAAA,EAAG,UAAA,EAAA;AACpB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,CAAC,WAAA,CAAY,GAAA,CAAI,EAAE,CAAA,EAAG;AAC1B,IAAA,IAAI,UAAA,GAAa,KAAK,UAAA,EAAY;AAGlC,IAAA,IAAI,OAAO,GAAA,EAAK;AACd,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA;AACvB,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA;AACvB,MAAA,IAAI,QAAQ,IAAA,IAAQ,OAAA,CAAQ,IAAI,CAAA,IAAK,OAAA,CAAQ,IAAI,CAAA,EAAG;AAEpD,MAAA,IAAI,IAAI,CAAA,GAAI,CAAA;AACZ,MAAA,OAAO,KAAK,CAAA,IAAK,WAAA,CAAY,KAAK,IAAA,CAAK,CAAC,CAAE,CAAA,EAAG,CAAA,EAAA;AAC7C,MAAA,MAAM,OAAO,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA,EAAG,CAAC,EAAE,WAAA,EAAY;AAC9C,MAAA,IAAI,MAAA,CAAO,GAAA,CAAI,IAAI,CAAA,EAAG;AAAA,IACxB;AAGA,IAAA,IAAI,IAAI,CAAA,GAAI,CAAA;AACZ,IAAA,OAAO,CAAA,GAAI,CAAA,KAAM,WAAA,CAAY,GAAA,CAAI,IAAA,CAAK,CAAC,CAAE,CAAA,IAAK,YAAA,CAAa,GAAA,CAAI,IAAA,CAAK,CAAC,CAAE,CAAA,IAAK,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,IAAO,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,IAAO,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,CAAA,EAAM;AAClI,MAAA,IAAI,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,eAAkB,CAAC,UAAA;AACnC,MAAA,CAAA,EAAA;AAAA,IACF;AAEA,IAAA,IAAI,KAAK,CAAA,IAAK,IAAA,CAAK,KAAK,IAAA,CAAK,CAAC,CAAE,CAAA,EAAG;AACjC,MAAA,IAAA,CAAK,OAAO,CAAC,CAAA;AACb,MAAA,KAAA,GAAQ,CAAA;AACR,MAAA,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACV;AAAA,EACF;AACA,EAAA,IAAI,KAAA,GAAQ,CAAA,EAAG,IAAA,CAAK,KAAA,EAAO,CAAC,CAAA;AAC5B,EAAA,OAAO,GAAA;AACT;;;ACDO,SAAS,QAAA,CAAS,OAAA,EAA2B,OAAA,GAA2B,EAAC,EAAmB;AACjG,EAAA,MAAM,WAAA,GAAc,QAAQ,WAAA,IAAe,IAAA;AAC3C,EAAA,MAAM,MAAA,GAAS,QAAQ,qBAAA,IAAyB,CAAA;AAChD,EAAA,MAAM,MAAA,GAAS,QAAQ,eAAA,IAAmB,GAAA;AAC1C,EAAA,MAAM,QAAA,GAAW,QAAQ,QAAA,IAAY,IAAA;AACrC,EAAA,MAAM,KAAA,GAAQ,QAAQ,WAAA,KAAgB,CAAC,MAAcA,qBAAA,CAAY,CAAA,EAAG,QAAQ,KAAK,CAAA,CAAA;AACjF,EAAA,MAAM,GAAA,GAAM,OAAA,CAAQ,SAAA,IAAa,OAAA,CAAQ,SAAA,CAAU,SAAS,IAAI,GAAA,CAAI,OAAA,CAAQ,SAAS,CAAA,GAAI,IAAA;AACzF,EAAA,MAAM,MAAA,GACJ,OAAA,CAAQ,MAAA,KAAW,CAAC,CAAA,KAAsC,CAAA,EAAA,EAAK,CAAA,CAAE,GAAA,GAAM,CAAC,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAA,CAAA;AAErF,EAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,EAAO,IAAA,EAAK,IAAK,CAAA,CAAA,EAAI,CAAA,GAAI,CAAC,CAAA,CAAE,CAAA;AAGnE,EAAA,MAAM,QAAqB,EAAC;AAC5B,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,GAAA,EAAK,IAAA,KAAS;AAC7B,IAAA,MAAM,KAAA,GAAQ,cAAA,CAAe,GAAA,CAAI,IAAA,IAAQ,EAAE,CAAA;AAC3C,IAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,GAAA,EAAK,QAAA,KAAa;AAC/B,MAAA,KAAA,CAAM,IAAA,CAAK;AAAA,QACT,MAAM,GAAA,CAAI,IAAA;AAAA,QACV,SAAA,EAAW,IAAA;AAAA,QACX,OAAO,GAAA,CAAI,KAAA;AAAA,QACX,KAAK,GAAA,CAAI,GAAA;AAAA,QACT,KAAA,EAAO,KAAA,EAAA;AAAA,QACP,KAAA,EAAO,CAAA;AAAA,QACP,MAAM,QAAA,KAAa,CAAA;AAAA,QACnB,MAAA,EAAQ,SAAA,CAAU,GAAA,CAAI,IAAI,CAAA;AAAA,QAC1B,KAAA,EAAO,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA;AAAA,QACxB,MAAA,EAAQ,SAAA,CAAU,GAAA,CAAI,IAAA,EAAM,GAAG,CAAA;AAAA,QAC/B,OAAA,EAAS,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA;AAAA,QAC1B,MAAA,EAAQ,KAAA,CAAM,GAAA,CAAI,IAAI;AAAA,OACvB,CAAA;AAAA,IACH,CAAC,CAAA;AAAA,EACH,CAAC,CAAA;AACD,EAAA,MAAM,iBAAiB,KAAA,CAAM,MAAA;AAC7B,EAAA,IAAI,mBAAmB,CAAA,EAAG;AACxB,IAAA,OAAO,EAAE,IAAA,EAAM,EAAA,EAAI,OAAA,EAAS,EAAC,EAAG,SAAA,EAAW,EAAC,EAAG,MAAA,EAAQ,CAAA,EAAG,UAAA,EAAY,CAAA,EAAG,gBAAgB,CAAA,EAAE;AAAA,EAC7F;AAGA,EAAA,MAAM,YAAA,GAAe,MAAM,GAAA,CAAI,CAAC,MAAM,CAAA,CAAE,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA;AACtD,EAAA,MAAM,MAAA,GAASC,WAAA;AAAA,IACb,YAAA;AAAA,IACA,KAAA,CAAM,GAAA,CAAI,CAAC,CAAA,MAAO,EAAE,EAAA,EAAI,MAAA,CAAO,CAAA,CAAE,KAAK,CAAA,EAAG,IAAA,EAAM,CAAA,CAAE,MAAK,CAAE;AAAA,GAC1D;AACA,EAAA,MAAM,SAAA,GAAY,IAAI,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAC,CAAA,CAAE,EAAA,EAAI,CAAA,CAAE,WAAW,CAAC,CAAC,CAAA;AAClE,EAAA,KAAA,MAAW,CAAA,IAAK,KAAA,EAAO,CAAA,CAAE,KAAA,GAAQ,SAAA,CAAU,IAAI,MAAA,CAAO,CAAA,CAAE,KAAK,CAAC,CAAA,IAAK,CAAA;AAInE,EAAA,MAAM,WAAW,CAAC,CAAA,KAAA,CACf,YAAY,CAAA,CAAE,IAAA,GAAO,IAAI,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,CAAA,GAAI,MAAM,CAAA,CAAE,MAAA,GAAS,IAAI,CAAA,CAAA,IAAM,CAAA,CAAE,SAAS,CAAA,GAAI,CAAA,CAAA;AAC1F,EAAA,MAAM,MAAA,GAAS,CAAC,GAAG,KAAK,EAAE,IAAA,CAAK,CAAC,GAAG,CAAA,KAAM;AACvC,IAAA,MAAM,EAAA,GAAK,SAAS,CAAC,CAAA;AACrB,IAAA,MAAM,EAAA,GAAK,SAAS,CAAC,CAAA;AACrB,IAAA,IAAI,EAAA,KAAO,EAAA,EAAI,OAAO,EAAA,GAAK,EAAA;AAC3B,IAAA,IAAI,EAAE,KAAA,KAAU,CAAA,CAAE,OAAO,OAAO,CAAA,CAAE,QAAQ,CAAA,CAAE,KAAA;AAC5C,IAAA,OAAO,CAAA,CAAE,QAAQ,CAAA,CAAE,KAAA;AAAA,EACrB,CAAC,CAAA;AAGD,EAAA,MAAM,OAAoB,EAAC;AAC3B,EAAA,MAAM,eAA8B,EAAC;AACrC,EAAA,MAAM,SAAA,uBAAgB,GAAA,EAAoB;AAC1C,EAAA,IAAI,IAAA,GAAO,CAAA;AACX,EAAA,IAAI,UAAA,GAAa,CAAA;AAEjB,EAAA,KAAA,MAAW,KAAK,MAAA,EAAQ;AACtB,IAAA,IAAI,GAAA,GAAM,KAAA;AACV,IAAA,KAAA,MAAW,MAAM,YAAA,EAAc;AAC7B,MAAA,IAAI,OAAA,CAAQ,CAAA,CAAE,OAAA,EAAS,EAAE,KAAK,MAAA,EAAQ;AACpC,QAAA,GAAA,GAAM,IAAA;AACN,QAAA;AAAA,MACF;AAAA,IACF;AACA,IAAA,IAAI,GAAA,EAAK;AACP,MAAA,UAAA,EAAA;AACA,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,UAAU,GAAA,CAAI,CAAA,CAAE,SAAS,CAAA,IAAK,MAAM,MAAA,EAAQ;AACjD,IAAA,IAAI,OAAO,CAAA,CAAE,MAAA,GAAS,WAAA,IAAe,IAAA,CAAK,SAAS,CAAA,EAAG;AACtD,IAAA,IAAA,CAAK,KAAK,CAAC,CAAA;AACX,IAAA,YAAA,CAAa,IAAA,CAAK,EAAE,OAAO,CAAA;AAC3B,IAAA,SAAA,CAAU,GAAA,CAAI,EAAE,SAAA,EAAA,CAAY,SAAA,CAAU,IAAI,CAAA,CAAE,SAAS,CAAA,IAAK,CAAA,IAAK,CAAC,CAAA;AAChE,IAAA,IAAA,IAAQ,CAAA,CAAE,MAAA;AAAA,EACZ;AAGA,EAAA,MAAM,QAAA,uBAAe,GAAA,EAAyB;AAC9C,EAAA,KAAA,MAAW,KAAK,IAAA,EAAM;AACpB,IAAA,MAAM,MAAM,QAAA,CAAS,GAAA,CAAI,CAAA,CAAE,SAAS,KAAK,EAAC;AAC1C,IAAA,GAAA,CAAI,KAAK,CAAC,CAAA;AACV,IAAA,QAAA,CAAS,GAAA,CAAI,CAAA,CAAE,SAAA,EAAW,GAAG,CAAA;AAAA,EAC/B;AAEA,EAAA,MAAM,KAAA,GAAQ,CAAC,CAAA,KAAoC;AACjD,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,IAAI,QAAA,IAAY,CAAA,CAAE,IAAA,EAAM,OAAA,CAAQ,KAAK,MAAM,CAAA;AAC3C,IAAA,IAAI,CAAA,CAAE,KAAA,EAAO,OAAA,CAAQ,IAAA,CAAK,OAAO,CAAA;AACjC,IAAA,IAAI,CAAA,CAAE,MAAA,EAAQ,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AACnC,IAAA,IAAI,CAAA,CAAE,MAAA,EAAQ,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AACnC,IAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAA,CAAQ,KAAK,MAAM,CAAA;AAC7C,IAAA,OAAO,EAAE,IAAA,EAAM,CAAA,CAAE,IAAA,EAAM,SAAA,EAAW,EAAE,SAAA,EAAW,KAAA,EAAO,CAAA,CAAE,KAAA,EAAO,KAAK,CAAA,CAAE,GAAA,EAAK,KAAA,EAAO,CAAA,CAAE,OAAO,OAAA,EAAQ;AAAA,EACrG,CAAA;AAEA,EAAA,MAAM,aAAgC,EAAC;AACvC,EAAA,MAAM,OAA4B,EAAC;AACnC,EAAA,MAAM,QAAkB,EAAC;AACzB,EAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,GAAA,EAAK,IAAA,KAAS;AAC7B,IAAA,MAAM,IAAA,GAAA,CAAQ,QAAA,CAAS,GAAA,CAAI,IAAI,KAAK,EAAC,EAAG,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,EAAE,KAAK,CAAA;AACxE,IAAA,IAAI,IAAA,CAAK,WAAW,CAAA,EAAG;AACvB,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,GAAA,CAAI,KAAK,CAAA;AAC/B,IAAA,UAAA,CAAW,IAAA,CAAK,EAAE,GAAA,EAAK,IAAA,EAAM,KAAA,EAAO,MAAA,CAAO,IAAI,CAAA,EAAI,GAAA,EAAK,GAAA,CAAI,GAAA,EAAK,SAAA,EAAW,UAAU,CAAA;AACtF,IAAA,IAAA,CAAK,IAAA,CAAK,GAAG,QAAQ,CAAA;AACrB,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,MAAA,CAAO,EAAE,KAAK,IAAA,EAAM,KAAA,EAAO,MAAA,CAAO,IAAI,CAAA,EAAI,GAAA,EAAK,GAAA,CAAI,GAAA,EAAK,CAAC;AAAA,EAAK,QAAA,CAAS,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAC,CAAA,CAAE,CAAA;AAAA,EACrH,CAAC,CAAA;AAED,EAAA,MAAM,IAAA,GAAO,KAAA,CAAM,IAAA,CAAK,MAAM,CAAA;AAC9B,EAAA,OAAO,EAAE,IAAA,EAAM,OAAA,EAAS,UAAA,EAAY,SAAA,EAAW,IAAA,EAAM,MAAA,EAAQ,KAAA,CAAM,IAAI,CAAA,EAAG,UAAA,EAAY,cAAA,EAAe;AACvG","file":"index.cjs","sourcesContent":["// Zero-dep detectors for the sentence features we always want to keep.\n\nconst CURRENCY = /(?:रु|नेरु|रुपैयाँ|₹|\\$|€|£|Rs\\.?|USD|NPR|INR)/i;\n// A number: Latin or Devanagari digits, optionally with separators/percent.\nconst NUMBERISH = /[0-9०-९][0-9०-९.,%]*/;\nconst PERCENT = /[%]|प्रतिशत/;\n// A quotation: matched straight or curly quotes with content between.\nconst QUOTED = /\"[^\"]{3,}\"|“[^”]{3,}”|‘[^’]{3,}’/;\n\nexport function hasNumber(text: string): boolean {\n  return NUMBERISH.test(text) || CURRENCY.test(text) || PERCENT.test(text);\n}\n\nexport function hasQuote(text: string): boolean {\n  return QUOTED.test(text);\n}\n\n/** True if any gazetteer term appears in the text (case-insensitive, whole-token where Latin). */\nexport function hasEntity(text: string, gaz: Set<string> | null): boolean {\n  if (!gaz || gaz.size === 0) return false;\n  const lower = text.toLowerCase();\n  for (const term of gaz) {\n    if (!term) continue;\n    // Devanagari has no case; Latin terms are matched with word boundaries.\n    if (/[^\\u0000-\\u007F]/.test(term)) {\n      if (text.includes(term)) return true;\n    } else if (new RegExp(`(?:^|[^a-z])${escapeRe(term.toLowerCase())}(?:[^a-z]|$)`).test(lower)) {\n      return true;\n    }\n  }\n  return false;\n}\n\nfunction escapeRe(s: string): string {\n  return s.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/** Word 3-gram shingles for near-duplicate detection. */\nexport function shingles(text: string, size = 3): Set<string> {\n  const words = text\n    .toLowerCase()\n    .replace(/[^\\p{L}\\p{N}\\s]/gu, \" \")\n    .split(/\\s+/)\n    .filter(Boolean);\n  const out = new Set<string>();\n  if (words.length < size) {\n    if (words.length) out.add(words.join(\" \"));\n    return out;\n  }\n  for (let i = 0; i + size <= words.length; i++) out.add(words.slice(i, i + size).join(\" \"));\n  return out;\n}\n\nexport function jaccard(a: Set<string>, b: Set<string>): number {\n  if (a.size === 0 && b.size === 0) return 0;\n  let inter = 0;\n  const [small, large] = a.size <= b.size ? [a, b] : [b, a];\n  for (const t of small) if (large.has(t)) inter++;\n  return inter / (a.size + b.size - inter);\n}\n","/** A sentence with its character offsets in the original source text. */\nexport interface Sentence {\n  text: string;\n  /** Inclusive start offset in the source. */\n  start: number;\n  /** Exclusive end offset in the source. */\n  end: number;\n}\n\n// Terminators: Latin . ! ? plus Devanagari danda । and double danda ॥ and the ellipsis.\nconst TERMINATORS = new Set([\".\", \"!\", \"?\", \"।\", \"॥\", \"…\"]);\n// Quote characters whose parity we track so we never split inside a quote.\nconst OPEN_QUOTES = new Set([\"“\", \"‘\", \"«\"]); // \" ' «\nconst CLOSE_QUOTES = new Set([\"”\", \"’\", \"»\"]); // \" ' »\n\n// Common abbreviations after which a period does NOT end a sentence.\nconst ABBREV = new Set([\n  \"mr\", \"mrs\", \"ms\", \"dr\", \"prof\", \"sr\", \"jr\", \"st\", \"vs\", \"etc\", \"inc\", \"ltd\",\n  \"co\", \"corp\", \"govt\", \"gen\", \"rep\", \"sen\", \"gov\", \"no\", \"vol\", \"fig\", \"al\",\n  \"rs\", \"u.s\", \"u.k\", \"e.g\", \"i.e\", \"a.m\", \"p.m\",\n]);\n\nfunction isDigit(ch: string): boolean {\n  return (ch >= \"0\" && ch <= \"9\") || (ch >= \"०\" && ch <= \"९\");\n}\n\n/**\n * Split `text` into sentences with character offsets. Handles Latin and\n * Devanagari terminators (। ॥), never splits inside a quotation, and does not\n * break on decimals (3.5), abbreviations (Dr.) or a terminator glued to a digit.\n * Deterministic: identical input always yields identical output.\n */\nexport function splitSentences(text: string): Sentence[] {\n  const out: Sentence[] = [];\n  if (!text) return out;\n  let quoteDepth = 0;\n  let doubleOpen = false;\n  let start = 0;\n  const n = text.length;\n\n  const push = (from: number, to: number) => {\n    const raw = text.slice(from, to);\n    const trimmedStart = raw.length - raw.trimStart().length;\n    const trimmedEnd = raw.length - raw.trimEnd().length;\n    const s = from + trimmedStart;\n    const e = to - trimmedEnd;\n    if (e > s) out.push({ text: text.slice(s, e), start: s, end: e });\n  };\n\n  for (let i = 0; i < n; i++) {\n    const ch = text[i]!;\n    if (ch === '\"') {\n      doubleOpen = !doubleOpen;\n      continue;\n    }\n    if (OPEN_QUOTES.has(ch)) {\n      quoteDepth++;\n      continue;\n    }\n    if (CLOSE_QUOTES.has(ch)) {\n      if (quoteDepth > 0) quoteDepth--;\n      continue;\n    }\n    if (!TERMINATORS.has(ch)) continue;\n    if (quoteDepth > 0 || doubleOpen) continue; // inside a quote — keep it whole\n\n    // A period between digits (3.5) or in an abbreviation is not a break.\n    if (ch === \".\") {\n      const prev = text[i - 1];\n      const next = text[i + 1];\n      if (prev && next && isDigit(prev) && isDigit(next)) continue;\n      // trailing abbreviation like \"Dr.\" — look back to the word\n      let j = i - 1;\n      while (j >= 0 && /[A-Za-z.]/.test(text[j]!)) j--;\n      const word = text.slice(j + 1, i).toLowerCase();\n      if (ABBREV.has(word)) continue;\n    }\n\n    // Consume any run of terminators/closing quotes/brackets.\n    let k = i + 1;\n    while (k < n && (TERMINATORS.has(text[k]!) || CLOSE_QUOTES.has(text[k]!) || text[k] === '\"' || text[k] === \")\" || text[k] === \"]\")) {\n      if (text[k] === '\"') doubleOpen = !doubleOpen;\n      k++;\n    }\n    // Must be followed by whitespace or end of text to count as a break.\n    if (k >= n || /\\s/.test(text[k]!)) {\n      push(start, k);\n      start = k;\n      i = k - 1;\n    }\n  }\n  if (start < n) push(start, n);\n  return out;\n}\n","import { bm25 } from \"@lacspace/rerank\";\nimport { countTokens } from \"@lacspace/tokenizer\";\nimport { hasEntity, hasNumber, hasQuote, jaccard, shingles } from \"./features.js\";\nimport { splitSentences } from \"./sentences.js\";\n\nexport { splitSentences } from \"./sentences.js\";\nexport type { Sentence } from \"./sentences.js\";\n\n/** One input article about the story. */\nexport interface CondenseSource {\n  text: string;\n  /** Short label used in the grouped output header, e.g. \"Kathmandu Post\". Defaults to \"S{n}\". */\n  label?: string;\n  url?: string;\n  publishedAt?: string | Date;\n}\n\nexport interface CondenseOptions {\n  /** Total token budget for the condensed digest. Default 1500. */\n  tokenBudget?: number;\n  /** Cap on kept sentences from any single source, so one long article can't crowd out the rest. Default 8. */\n  maxSentencesPerSource?: number;\n  /** Drop a sentence whose 3-gram Jaccard similarity to an already-kept sentence is ≥ this. Default 0.8. */\n  dedupeThreshold?: number;\n  /** Always keep each source's first sentence (the lede) regardless of score. Default true. */\n  keepLede?: boolean;\n  /** Named entities (people/places) to always keep and to boost — pass en and ne forms. */\n  gazetteer?: string[];\n  /** Model hint for token counting (passed to @lacspace/tokenizer). */\n  model?: string;\n  /** Override token counting entirely. Default: @lacspace/tokenizer countTokens. */\n  countTokens?: (text: string) => number;\n  /** Header format for each source group. Default `[S{n} {label}]`. */\n  header?: (source: { idx: number; label: string; url?: string }) => string;\n}\n\nexport interface CondensedSentence {\n  text: string;\n  /** Index of the source this sentence came from (0-based). */\n  sourceIdx: number;\n  /** Character offsets in that source's original text. */\n  start: number;\n  end: number;\n  /** BM25 centrality score within the cluster. */\n  score: number;\n  /** Why it was kept: any of \"lede\", \"number\", \"quote\", \"entity\", \"rank\". */\n  reasons: string[];\n}\n\nexport interface CondensedSource {\n  idx: number;\n  label: string;\n  url?: string;\n  sentences: CondensedSentence[];\n}\n\nexport interface CondenseResult {\n  /** The digest: kept sentences grouped per source (source order) under a header line. */\n  text: string;\n  /** Kept sentences grouped per source, each in the source's original order. */\n  sources: CondensedSource[];\n  /** All kept sentences, flat, in output (source-grouped) order. */\n  sentences: CondensedSentence[];\n  /** Token count of `text` (same counter used for the budget). */\n  tokens: number;\n  /** How many near-duplicate sentences were dropped. */\n  droppedDup: number;\n  /** Total sentences seen across all sources. */\n  totalSentences: number;\n}\n\ninterface Candidate {\n  text: string;\n  sourceIdx: number;\n  start: number;\n  end: number;\n  order: number; // global index for stable ties\n  score: number;\n  lede: boolean;\n  number: boolean;\n  quote: boolean;\n  entity: boolean;\n  shingle: Set<string>;\n  tokens: number;\n}\n\n/**\n * Condense several sources on the same story into one short, deduplicated,\n * token-budgeted digest that keeps the numbers, quotes and named entities — so\n * an LLM only rewrites a fraction of the words. Purely extractive and\n * deterministic; no network, no model.\n */\nexport function condense(sources: CondenseSource[], options: CondenseOptions = {}): CondenseResult {\n  const tokenBudget = options.tokenBudget ?? 1500;\n  const maxPer = options.maxSentencesPerSource ?? 8;\n  const dedupe = options.dedupeThreshold ?? 0.8;\n  const keepLede = options.keepLede ?? true;\n  const count = options.countTokens ?? ((t: string) => countTokens(t, options.model));\n  const gaz = options.gazetteer && options.gazetteer.length ? new Set(options.gazetteer) : null;\n  const header =\n    options.header ?? ((s: { idx: number; label: string }) => `[S${s.idx + 1} ${s.label}]`);\n\n  const labels = sources.map((s, i) => s.label?.trim() || `S${i + 1}`);\n\n  // 1. Split every source into sentences with offsets.\n  const cands: Candidate[] = [];\n  let order = 0;\n  sources.forEach((src, sIdx) => {\n    const sents = splitSentences(src.text ?? \"\");\n    sents.forEach((sen, localIdx) => {\n      cands.push({\n        text: sen.text,\n        sourceIdx: sIdx,\n        start: sen.start,\n        end: sen.end,\n        order: order++,\n        score: 0,\n        lede: localIdx === 0,\n        number: hasNumber(sen.text),\n        quote: hasQuote(sen.text),\n        entity: hasEntity(sen.text, gaz),\n        shingle: shingles(sen.text),\n        tokens: count(sen.text),\n      });\n    });\n  });\n  const totalSentences = cands.length;\n  if (totalSentences === 0) {\n    return { text: \"\", sources: [], sentences: [], tokens: 0, droppedDup: 0, totalSentences: 0 };\n  }\n\n  // 2. Score sentence centrality with BM25 against the whole cluster.\n  const clusterQuery = cands.map((c) => c.text).join(\" \");\n  const scored = bm25(\n    clusterQuery,\n    cands.map((c) => ({ id: String(c.order), text: c.text })),\n  );\n  const scoreById = new Map(scored.map((s) => [s.id, s.rerankScore]));\n  for (const c of cands) c.score = scoreById.get(String(c.order)) ?? 0;\n\n  // 3. Priority: always-keep (lede/number/quote/entity) first, then by score.\n  //    Deterministic tie-break by global order.\n  const priority = (c: Candidate) =>\n    (keepLede && c.lede ? 4 : 0) + (c.quote ? 3 : 0) + (c.number ? 2 : 0) + (c.entity ? 1 : 0);\n  const ranked = [...cands].sort((a, b) => {\n    const pa = priority(a);\n    const pb = priority(b);\n    if (pa !== pb) return pb - pa;\n    if (b.score !== a.score) return b.score - a.score;\n    return a.order - b.order;\n  });\n\n  // 4. Greedy selection: dedupe against kept, respect per-source cap and budget.\n  const kept: Candidate[] = [];\n  const keptShingles: Set<string>[] = [];\n  const perSource = new Map<number, number>();\n  let used = 0;\n  let droppedDup = 0;\n\n  for (const c of ranked) {\n    let dup = false;\n    for (const ks of keptShingles) {\n      if (jaccard(c.shingle, ks) >= dedupe) {\n        dup = true;\n        break;\n      }\n    }\n    if (dup) {\n      droppedDup++;\n      continue;\n    }\n    if ((perSource.get(c.sourceIdx) ?? 0) >= maxPer) continue;\n    if (used + c.tokens > tokenBudget && kept.length > 0) continue; // keep at least one\n    kept.push(c);\n    keptShingles.push(c.shingle);\n    perSource.set(c.sourceIdx, (perSource.get(c.sourceIdx) ?? 0) + 1);\n    used += c.tokens;\n  }\n\n  // 5. Group kept sentences per source, each in original reading order.\n  const bySource = new Map<number, Candidate[]>();\n  for (const c of kept) {\n    const arr = bySource.get(c.sourceIdx) ?? [];\n    arr.push(c);\n    bySource.set(c.sourceIdx, arr);\n  }\n\n  const toOut = (c: Candidate): CondensedSentence => {\n    const reasons: string[] = [];\n    if (keepLede && c.lede) reasons.push(\"lede\");\n    if (c.quote) reasons.push(\"quote\");\n    if (c.number) reasons.push(\"number\");\n    if (c.entity) reasons.push(\"entity\");\n    if (reasons.length === 0) reasons.push(\"rank\");\n    return { text: c.text, sourceIdx: c.sourceIdx, start: c.start, end: c.end, score: c.score, reasons };\n  };\n\n  const outSources: CondensedSource[] = [];\n  const flat: CondensedSentence[] = [];\n  const parts: string[] = [];\n  sources.forEach((src, sIdx) => {\n    const list = (bySource.get(sIdx) ?? []).sort((a, b) => a.start - b.start);\n    if (list.length === 0) return;\n    const outSents = list.map(toOut);\n    outSources.push({ idx: sIdx, label: labels[sIdx]!, url: src.url, sentences: outSents });\n    flat.push(...outSents);\n    parts.push(`${header({ idx: sIdx, label: labels[sIdx]!, url: src.url })}\\n${outSents.map((s) => s.text).join(\" \")}`);\n  });\n\n  const text = parts.join(\"\\n\\n\");\n  return { text, sources: outSources, sentences: flat, tokens: count(text), droppedDup, totalSentences };\n}\n"]}

package/dist/index.d.cts

@@ -0,0 +1,87 @@
+/** A sentence with its character offsets in the original source text. */
+interface Sentence {
+    text: string;
+    /** Inclusive start offset in the source. */
+    start: number;
+    /** Exclusive end offset in the source. */
+    end: number;
+}
+/**
+ * Split `text` into sentences with character offsets. Handles Latin and
+ * Devanagari terminators (। ॥), never splits inside a quotation, and does not
+ * break on decimals (3.5), abbreviations (Dr.) or a terminator glued to a digit.
+ * Deterministic: identical input always yields identical output.
+ */
+declare function splitSentences(text: string): Sentence[];
+
+/** One input article about the story. */
+interface CondenseSource {
+    text: string;
+    /** Short label used in the grouped output header, e.g. "Kathmandu Post". Defaults to "S{n}". */
+    label?: string;
+    url?: string;
+    publishedAt?: string | Date;
+}
+interface CondenseOptions {
+    /** Total token budget for the condensed digest. Default 1500. */
+    tokenBudget?: number;
+    /** Cap on kept sentences from any single source, so one long article can't crowd out the rest. Default 8. */
+    maxSentencesPerSource?: number;
+    /** Drop a sentence whose 3-gram Jaccard similarity to an already-kept sentence is ≥ this. Default 0.8. */
+    dedupeThreshold?: number;
+    /** Always keep each source's first sentence (the lede) regardless of score. Default true. */
+    keepLede?: boolean;
+    /** Named entities (people/places) to always keep and to boost — pass en and ne forms. */
+    gazetteer?: string[];
+    /** Model hint for token counting (passed to @lacspace/tokenizer). */
+    model?: string;
+    /** Override token counting entirely. Default: @lacspace/tokenizer countTokens. */
+    countTokens?: (text: string) => number;
+    /** Header format for each source group. Default `[S{n} {label}]`. */
+    header?: (source: {
+        idx: number;
+        label: string;
+        url?: string;
+    }) => string;
+}
+interface CondensedSentence {
+    text: string;
+    /** Index of the source this sentence came from (0-based). */
+    sourceIdx: number;
+    /** Character offsets in that source's original text. */
+    start: number;
+    end: number;
+    /** BM25 centrality score within the cluster. */
+    score: number;
+    /** Why it was kept: any of "lede", "number", "quote", "entity", "rank". */
+    reasons: string[];
+}
+interface CondensedSource {
+    idx: number;
+    label: string;
+    url?: string;
+    sentences: CondensedSentence[];
+}
+interface CondenseResult {
+    /** The digest: kept sentences grouped per source (source order) under a header line. */
+    text: string;
+    /** Kept sentences grouped per source, each in the source's original order. */
+    sources: CondensedSource[];
+    /** All kept sentences, flat, in output (source-grouped) order. */
+    sentences: CondensedSentence[];
+    /** Token count of `text` (same counter used for the budget). */
+    tokens: number;
+    /** How many near-duplicate sentences were dropped. */
+    droppedDup: number;
+    /** Total sentences seen across all sources. */
+    totalSentences: number;
+}
+/**
+ * Condense several sources on the same story into one short, deduplicated,
+ * token-budgeted digest that keeps the numbers, quotes and named entities — so
+ * an LLM only rewrites a fraction of the words. Purely extractive and
+ * deterministic; no network, no model.
+ */
+declare function condense(sources: CondenseSource[], options?: CondenseOptions): CondenseResult;
+
+export { type CondenseOptions, type CondenseResult, type CondenseSource, type CondensedSentence, type CondensedSource, type Sentence, condense, splitSentences };

package/dist/index.d.ts

@@ -0,0 +1,87 @@
+/** A sentence with its character offsets in the original source text. */
+interface Sentence {
+    text: string;
+    /** Inclusive start offset in the source. */
+    start: number;
+    /** Exclusive end offset in the source. */
+    end: number;
+}
+/**
+ * Split `text` into sentences with character offsets. Handles Latin and
+ * Devanagari terminators (। ॥), never splits inside a quotation, and does not
+ * break on decimals (3.5), abbreviations (Dr.) or a terminator glued to a digit.
+ * Deterministic: identical input always yields identical output.
+ */
+declare function splitSentences(text: string): Sentence[];
+
+/** One input article about the story. */
+interface CondenseSource {
+    text: string;
+    /** Short label used in the grouped output header, e.g. "Kathmandu Post". Defaults to "S{n}". */
+    label?: string;
+    url?: string;
+    publishedAt?: string | Date;
+}
+interface CondenseOptions {
+    /** Total token budget for the condensed digest. Default 1500. */
+    tokenBudget?: number;
+    /** Cap on kept sentences from any single source, so one long article can't crowd out the rest. Default 8. */
+    maxSentencesPerSource?: number;
+    /** Drop a sentence whose 3-gram Jaccard similarity to an already-kept sentence is ≥ this. Default 0.8. */
+    dedupeThreshold?: number;
+    /** Always keep each source's first sentence (the lede) regardless of score. Default true. */
+    keepLede?: boolean;
+    /** Named entities (people/places) to always keep and to boost — pass en and ne forms. */
+    gazetteer?: string[];
+    /** Model hint for token counting (passed to @lacspace/tokenizer). */
+    model?: string;
+    /** Override token counting entirely. Default: @lacspace/tokenizer countTokens. */
+    countTokens?: (text: string) => number;
+    /** Header format for each source group. Default `[S{n} {label}]`. */
+    header?: (source: {
+        idx: number;
+        label: string;
+        url?: string;
+    }) => string;
+}
+interface CondensedSentence {
+    text: string;
+    /** Index of the source this sentence came from (0-based). */
+    sourceIdx: number;
+    /** Character offsets in that source's original text. */
+    start: number;
+    end: number;
+    /** BM25 centrality score within the cluster. */
+    score: number;
+    /** Why it was kept: any of "lede", "number", "quote", "entity", "rank". */
+    reasons: string[];
+}
+interface CondensedSource {
+    idx: number;
+    label: string;
+    url?: string;
+    sentences: CondensedSentence[];
+}
+interface CondenseResult {
+    /** The digest: kept sentences grouped per source (source order) under a header line. */
+    text: string;
+    /** Kept sentences grouped per source, each in the source's original order. */
+    sources: CondensedSource[];
+    /** All kept sentences, flat, in output (source-grouped) order. */
+    sentences: CondensedSentence[];
+    /** Token count of `text` (same counter used for the budget). */
+    tokens: number;
+    /** How many near-duplicate sentences were dropped. */
+    droppedDup: number;
+    /** Total sentences seen across all sources. */
+    totalSentences: number;
+}
+/**
+ * Condense several sources on the same story into one short, deduplicated,
+ * token-budgeted digest that keeps the numbers, quotes and named entities — so
+ * an LLM only rewrites a fraction of the words. Purely extractive and
+ * deterministic; no network, no model.
+ */
+declare function condense(sources: CondenseSource[], options?: CondenseOptions): CondenseResult;
+
+export { type CondenseOptions, type CondenseResult, type CondenseSource, type CondensedSentence, type CondensedSource, type Sentence, condense, splitSentences };

package/dist/index.js

@@ -0,0 +1,252 @@
+import { bm25 } from '@lacspace/rerank';
+import { countTokens } from '@lacspace/tokenizer';
+
+// src/index.ts
+
+// src/features.ts
+var CURRENCY = /(?:रु|नेरु|रुपैयाँ|₹|\$|€|£|Rs\.?|USD|NPR|INR)/i;
+var NUMBERISH = /[0-9०-९][0-9०-९.,%]*/;
+var PERCENT = /[%]|प्रतिशत/;
+var QUOTED = /"[^"]{3,}"|“[^”]{3,}”|‘[^’]{3,}’/;
+function hasNumber(text) {
+  return NUMBERISH.test(text) || CURRENCY.test(text) || PERCENT.test(text);
+}
+function hasQuote(text) {
+  return QUOTED.test(text);
+}
+function hasEntity(text, gaz) {
+  if (!gaz || gaz.size === 0) return false;
+  const lower = text.toLowerCase();
+  for (const term of gaz) {
+    if (!term) continue;
+    if (/[^\u0000-\u007F]/.test(term)) {
+      if (text.includes(term)) return true;
+    } else if (new RegExp(`(?:^|[^a-z])${escapeRe(term.toLowerCase())}(?:[^a-z]|$)`).test(lower)) {
+      return true;
+    }
+  }
+  return false;
+}
+function escapeRe(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function shingles(text, size = 3) {
+  const words = text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter(Boolean);
+  const out = /* @__PURE__ */ new Set();
+  if (words.length < size) {
+    if (words.length) out.add(words.join(" "));
+    return out;
+  }
+  for (let i = 0; i + size <= words.length; i++) out.add(words.slice(i, i + size).join(" "));
+  return out;
+}
+function jaccard(a, b) {
+  if (a.size === 0 && b.size === 0) return 0;
+  let inter = 0;
+  const [small, large] = a.size <= b.size ? [a, b] : [b, a];
+  for (const t of small) if (large.has(t)) inter++;
+  return inter / (a.size + b.size - inter);
+}
+
+// src/sentences.ts
+var TERMINATORS = /* @__PURE__ */ new Set([".", "!", "?", "\u0964", "\u0965", "\u2026"]);
+var OPEN_QUOTES = /* @__PURE__ */ new Set(["\u201C", "\u2018", "\xAB"]);
+var CLOSE_QUOTES = /* @__PURE__ */ new Set(["\u201D", "\u2019", "\xBB"]);
+var ABBREV = /* @__PURE__ */ new Set([
+  "mr",
+  "mrs",
+  "ms",
+  "dr",
+  "prof",
+  "sr",
+  "jr",
+  "st",
+  "vs",
+  "etc",
+  "inc",
+  "ltd",
+  "co",
+  "corp",
+  "govt",
+  "gen",
+  "rep",
+  "sen",
+  "gov",
+  "no",
+  "vol",
+  "fig",
+  "al",
+  "rs",
+  "u.s",
+  "u.k",
+  "e.g",
+  "i.e",
+  "a.m",
+  "p.m"
+]);
+function isDigit(ch) {
+  return ch >= "0" && ch <= "9" || ch >= "\u0966" && ch <= "\u096F";
+}
+function splitSentences(text) {
+  const out = [];
+  if (!text) return out;
+  let quoteDepth = 0;
+  let doubleOpen = false;
+  let start = 0;
+  const n = text.length;
+  const push = (from, to) => {
+    const raw = text.slice(from, to);
+    const trimmedStart = raw.length - raw.trimStart().length;
+    const trimmedEnd = raw.length - raw.trimEnd().length;
+    const s = from + trimmedStart;
+    const e = to - trimmedEnd;
+    if (e > s) out.push({ text: text.slice(s, e), start: s, end: e });
+  };
+  for (let i = 0; i < n; i++) {
+    const ch = text[i];
+    if (ch === '"') {
+      doubleOpen = !doubleOpen;
+      continue;
+    }
+    if (OPEN_QUOTES.has(ch)) {
+      quoteDepth++;
+      continue;
+    }
+    if (CLOSE_QUOTES.has(ch)) {
+      if (quoteDepth > 0) quoteDepth--;
+      continue;
+    }
+    if (!TERMINATORS.has(ch)) continue;
+    if (quoteDepth > 0 || doubleOpen) continue;
+    if (ch === ".") {
+      const prev = text[i - 1];
+      const next = text[i + 1];
+      if (prev && next && isDigit(prev) && isDigit(next)) continue;
+      let j = i - 1;
+      while (j >= 0 && /[A-Za-z.]/.test(text[j])) j--;
+      const word = text.slice(j + 1, i).toLowerCase();
+      if (ABBREV.has(word)) continue;
+    }
+    let k = i + 1;
+    while (k < n && (TERMINATORS.has(text[k]) || CLOSE_QUOTES.has(text[k]) || text[k] === '"' || text[k] === ")" || text[k] === "]")) {
+      if (text[k] === '"') doubleOpen = !doubleOpen;
+      k++;
+    }
+    if (k >= n || /\s/.test(text[k])) {
+      push(start, k);
+      start = k;
+      i = k - 1;
+    }
+  }
+  if (start < n) push(start, n);
+  return out;
+}
+
+// src/index.ts
+function condense(sources, options = {}) {
+  const tokenBudget = options.tokenBudget ?? 1500;
+  const maxPer = options.maxSentencesPerSource ?? 8;
+  const dedupe = options.dedupeThreshold ?? 0.8;
+  const keepLede = options.keepLede ?? true;
+  const count = options.countTokens ?? ((t) => countTokens(t, options.model));
+  const gaz = options.gazetteer && options.gazetteer.length ? new Set(options.gazetteer) : null;
+  const header = options.header ?? ((s) => `[S${s.idx + 1} ${s.label}]`);
+  const labels = sources.map((s, i) => s.label?.trim() || `S${i + 1}`);
+  const cands = [];
+  let order = 0;
+  sources.forEach((src, sIdx) => {
+    const sents = splitSentences(src.text ?? "");
+    sents.forEach((sen, localIdx) => {
+      cands.push({
+        text: sen.text,
+        sourceIdx: sIdx,
+        start: sen.start,
+        end: sen.end,
+        order: order++,
+        score: 0,
+        lede: localIdx === 0,
+        number: hasNumber(sen.text),
+        quote: hasQuote(sen.text),
+        entity: hasEntity(sen.text, gaz),
+        shingle: shingles(sen.text),
+        tokens: count(sen.text)
+      });
+    });
+  });
+  const totalSentences = cands.length;
+  if (totalSentences === 0) {
+    return { text: "", sources: [], sentences: [], tokens: 0, droppedDup: 0, totalSentences: 0 };
+  }
+  const clusterQuery = cands.map((c) => c.text).join(" ");
+  const scored = bm25(
+    clusterQuery,
+    cands.map((c) => ({ id: String(c.order), text: c.text }))
+  );
+  const scoreById = new Map(scored.map((s) => [s.id, s.rerankScore]));
+  for (const c of cands) c.score = scoreById.get(String(c.order)) ?? 0;
+  const priority = (c) => (keepLede && c.lede ? 4 : 0) + (c.quote ? 3 : 0) + (c.number ? 2 : 0) + (c.entity ? 1 : 0);
+  const ranked = [...cands].sort((a, b) => {
+    const pa = priority(a);
+    const pb = priority(b);
+    if (pa !== pb) return pb - pa;
+    if (b.score !== a.score) return b.score - a.score;
+    return a.order - b.order;
+  });
+  const kept = [];
+  const keptShingles = [];
+  const perSource = /* @__PURE__ */ new Map();
+  let used = 0;
+  let droppedDup = 0;
+  for (const c of ranked) {
+    let dup = false;
+    for (const ks of keptShingles) {
+      if (jaccard(c.shingle, ks) >= dedupe) {
+        dup = true;
+        break;
+      }
+    }
+    if (dup) {
+      droppedDup++;
+      continue;
+    }
+    if ((perSource.get(c.sourceIdx) ?? 0) >= maxPer) continue;
+    if (used + c.tokens > tokenBudget && kept.length > 0) continue;
+    kept.push(c);
+    keptShingles.push(c.shingle);
+    perSource.set(c.sourceIdx, (perSource.get(c.sourceIdx) ?? 0) + 1);
+    used += c.tokens;
+  }
+  const bySource = /* @__PURE__ */ new Map();
+  for (const c of kept) {
+    const arr = bySource.get(c.sourceIdx) ?? [];
+    arr.push(c);
+    bySource.set(c.sourceIdx, arr);
+  }
+  const toOut = (c) => {
+    const reasons = [];
+    if (keepLede && c.lede) reasons.push("lede");
+    if (c.quote) reasons.push("quote");
+    if (c.number) reasons.push("number");
+    if (c.entity) reasons.push("entity");
+    if (reasons.length === 0) reasons.push("rank");
+    return { text: c.text, sourceIdx: c.sourceIdx, start: c.start, end: c.end, score: c.score, reasons };
+  };
+  const outSources = [];
+  const flat = [];
+  const parts = [];
+  sources.forEach((src, sIdx) => {
+    const list = (bySource.get(sIdx) ?? []).sort((a, b) => a.start - b.start);
+    if (list.length === 0) return;
+    const outSents = list.map(toOut);
+    outSources.push({ idx: sIdx, label: labels[sIdx], url: src.url, sentences: outSents });
+    flat.push(...outSents);
+    parts.push(`${header({ idx: sIdx, label: labels[sIdx], url: src.url })}
+${outSents.map((s) => s.text).join(" ")}`);
+  });
+  const text = parts.join("\n\n");
+  return { text, sources: outSources, sentences: flat, tokens: count(text), droppedDup, totalSentences };
+}
+
+export { condense, splitSentences };
+//# sourceMappingURL=index.js.map
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/features.ts","../src/sentences.ts","../src/index.ts"],"names":[],"mappings":";;;;;;AAEA,IAAM,QAAA,GAAW,iDAAA;AAEjB,IAAM,SAAA,GAAY,sBAAA;AAClB,IAAM,OAAA,GAAU,aAAA;AAEhB,IAAM,MAAA,GAAS,kCAAA;AAER,SAAS,UAAU,IAAA,EAAuB;AAC/C,EAAA,OAAO,SAAA,CAAU,IAAA,CAAK,IAAI,CAAA,IAAK,QAAA,CAAS,KAAK,IAAI,CAAA,IAAK,OAAA,CAAQ,IAAA,CAAK,IAAI,CAAA;AACzE;AAEO,SAAS,SAAS,IAAA,EAAuB;AAC9C,EAAA,OAAO,MAAA,CAAO,KAAK,IAAI,CAAA;AACzB;AAGO,SAAS,SAAA,CAAU,MAAc,GAAA,EAAkC;AACxE,EAAA,IAAI,CAAC,GAAA,IAAO,GAAA,CAAI,IAAA,KAAS,GAAG,OAAO,KAAA;AACnC,EAAA,MAAM,KAAA,GAAQ,KAAK,WAAA,EAAY;AAC/B,EAAA,KAAA,MAAW,QAAQ,GAAA,EAAK;AACtB,IAAA,IAAI,CAAC,IAAA,EAAM;AAEX,IAAA,IAAI,kBAAA,CAAmB,IAAA,CAAK,IAAI,CAAA,EAAG;AACjC,MAAA,IAAI,IAAA,CAAK,QAAA,CAAS,IAAI,CAAA,EAAG,OAAO,IAAA;AAAA,IAClC,CAAA,MAAA,IAAW,IAAI,MAAA,CAAO,CAAA,YAAA,EAAe,QAAA,CAAS,IAAA,CAAK,WAAA,EAAa,CAAC,CAAA,YAAA,CAAc,CAAA,CAAE,IAAA,CAAK,KAAK,CAAA,EAAG;AAC5F,MAAA,OAAO,IAAA;AAAA,IACT;AAAA,EACF;AACA,EAAA,OAAO,KAAA;AACT;AAEA,SAAS,SAAS,CAAA,EAAmB;AACnC,EAAA,OAAO,CAAA,CAAE,OAAA,CAAQ,qBAAA,EAAuB,MAAM,CAAA;AAChD;AAGO,SAAS,QAAA,CAAS,IAAA,EAAc,IAAA,GAAO,CAAA,EAAgB;AAC5D,EAAA,MAAM,KAAA,GAAQ,IAAA,CACX,WAAA,EAAY,CACZ,OAAA,CAAQ,mBAAA,EAAqB,GAAG,CAAA,CAChC,KAAA,CAAM,KAAK,CAAA,CACX,MAAA,CAAO,OAAO,CAAA;AACjB,EAAA,MAAM,GAAA,uBAAU,GAAA,EAAY;AAC5B,EAAA,IAAI,KAAA,CAAM,SAAS,IAAA,EAAM;AACvB,IAAA,IAAI,MAAM,MAAA,EAAQ,GAAA,CAAI,IAAI,KAAA,CAAM,IAAA,CAAK,GAAG,CAAC,CAAA;AACzC,IAAA,OAAO,GAAA;AAAA,EACT;AACA,EAAA,KAAA,IAAS,IAAI,CAAA,EAAG,CAAA,GAAI,IAAA,IAAQ,KAAA,CAAM,QAAQ,CAAA,EAAA,EAAK,GAAA,CAAI,GAAA,CAAI,KAAA,CAAM,MAAM,CAAA,EAAG,CAAA,GAAI,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAC,CAAA;AACzF,EAAA,OAAO,GAAA;AACT;AAEO,SAAS,OAAA,CAAQ,GAAgB,CAAA,EAAwB;AAC9D,EAAA,IAAI,EAAE,IAAA,KAAS,CAAA,IAAK,CAAA,CAAE,IAAA,KAAS,GAAG,OAAO,CAAA;AACzC,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,MAAM,CAAC,KAAA,EAAO,KAAK,CAAA,GAAI,EAAE,IAAA,IAAQ,CAAA,CAAE,IAAA,GAAO,CAAC,CAAA,EAAG,CAAC,CAAA,GAAI,CAAC,GAAG,CAAC,CAAA;AACxD,EAAA,KAAA,MAAW,KAAK,KAAA,EAAO,IAAI,KAAA,CAAM,GAAA,CAAI,CAAC,CAAA,EAAG,KAAA,EAAA;AACzC,EAAA,OAAO,KAAA,IAAS,CAAA,CAAE,IAAA,GAAO,CAAA,CAAE,IAAA,GAAO,KAAA,CAAA;AACpC;;;ACjDA,IAAM,WAAA,mBAAc,IAAI,GAAA,CAAI,CAAC,GAAA,EAAK,KAAK,GAAA,EAAK,QAAA,EAAK,QAAA,EAAK,QAAG,CAAC,CAAA;AAE1D,IAAM,8BAAc,IAAI,GAAA,CAAI,CAAC,QAAA,EAAK,QAAA,EAAK,MAAG,CAAC,CAAA;AAC3C,IAAM,+BAAe,IAAI,GAAA,CAAI,CAAC,QAAA,EAAK,QAAA,EAAK,MAAG,CAAC,CAAA;AAG5C,IAAM,MAAA,uBAAa,GAAA,CAAI;AAAA,EACrB,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,MAAA;AAAA,EAAQ,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EACvE,IAAA;AAAA,EAAM,MAAA;AAAA,EAAQ,MAAA;AAAA,EAAQ,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,IAAA;AAAA,EACtE,IAAA;AAAA,EAAM,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO,KAAA;AAAA,EAAO;AAC3C,CAAC,CAAA;AAED,SAAS,QAAQ,EAAA,EAAqB;AACpC,EAAA,OAAQ,MAAM,GAAA,IAAO,EAAA,IAAM,GAAA,IAAS,EAAA,IAAM,YAAO,EAAA,IAAM,QAAA;AACzD;AAQO,SAAS,eAAe,IAAA,EAA0B;AACvD,EAAA,MAAM,MAAkB,EAAC;AACzB,EAAA,IAAI,CAAC,MAAM,OAAO,GAAA;AAClB,EAAA,IAAI,UAAA,GAAa,CAAA;AACjB,EAAA,IAAI,UAAA,GAAa,KAAA;AACjB,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,MAAM,IAAI,IAAA,CAAK,MAAA;AAEf,EAAA,MAAM,IAAA,GAAO,CAAC,IAAA,EAAc,EAAA,KAAe;AACzC,IAAA,MAAM,GAAA,GAAM,IAAA,CAAK,KAAA,CAAM,IAAA,EAAM,EAAE,CAAA;AAC/B,IAAA,MAAM,YAAA,GAAe,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,WAAU,CAAE,MAAA;AAClD,IAAA,MAAM,UAAA,GAAa,GAAA,CAAI,MAAA,GAAS,GAAA,CAAI,SAAQ,CAAE,MAAA;AAC9C,IAAA,MAAM,IAAI,IAAA,GAAO,YAAA;AACjB,IAAA,MAAM,IAAI,EAAA,GAAK,UAAA;AACf,IAAA,IAAI,CAAA,GAAI,CAAA,EAAG,GAAA,CAAI,IAAA,CAAK,EAAE,IAAA,EAAM,IAAA,CAAK,KAAA,CAAM,CAAA,EAAG,CAAC,CAAA,EAAG,KAAA,EAAO,CAAA,EAAG,GAAA,EAAK,GAAG,CAAA;AAAA,EAClE,CAAA;AAEA,EAAA,KAAA,IAAS,CAAA,GAAI,CAAA,EAAG,CAAA,GAAI,CAAA,EAAG,CAAA,EAAA,EAAK;AAC1B,IAAA,MAAM,EAAA,GAAK,KAAK,CAAC,CAAA;AACjB,IAAA,IAAI,OAAO,GAAA,EAAK;AACd,MAAA,UAAA,GAAa,CAAC,UAAA;AACd,MAAA;AAAA,IACF;AACA,IAAA,IAAI,WAAA,CAAY,GAAA,CAAI,EAAE,CAAA,EAAG;AACvB,MAAA,UAAA,EAAA;AACA,MAAA;AAAA,IACF;AACA,IAAA,IAAI,YAAA,CAAa,GAAA,CAAI,EAAE,CAAA,EAAG;AACxB,MAAA,IAAI,aAAa,CAAA,EAAG,UAAA,EAAA;AACpB,MAAA;AAAA,IACF;AACA,IAAA,IAAI,CAAC,WAAA,CAAY,GAAA,CAAI,EAAE,CAAA,EAAG;AAC1B,IAAA,IAAI,UAAA,GAAa,KAAK,UAAA,EAAY;AAGlC,IAAA,IAAI,OAAO,GAAA,EAAK;AACd,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA;AACvB,MAAA,MAAM,IAAA,GAAO,IAAA,CAAK,CAAA,GAAI,CAAC,CAAA;AACvB,MAAA,IAAI,QAAQ,IAAA,IAAQ,OAAA,CAAQ,IAAI,CAAA,IAAK,OAAA,CAAQ,IAAI,CAAA,EAAG;AAEpD,MAAA,IAAI,IAAI,CAAA,GAAI,CAAA;AACZ,MAAA,OAAO,KAAK,CAAA,IAAK,WAAA,CAAY,KAAK,IAAA,CAAK,CAAC,CAAE,CAAA,EAAG,CAAA,EAAA;AAC7C,MAAA,MAAM,OAAO,IAAA,CAAK,KAAA,CAAM,IAAI,CAAA,EAAG,CAAC,EAAE,WAAA,EAAY;AAC9C,MAAA,IAAI,MAAA,CAAO,GAAA,CAAI,IAAI,CAAA,EAAG;AAAA,IACxB;AAGA,IAAA,IAAI,IAAI,CAAA,GAAI,CAAA;AACZ,IAAA,OAAO,CAAA,GAAI,CAAA,KAAM,WAAA,CAAY,GAAA,CAAI,IAAA,CAAK,CAAC,CAAE,CAAA,IAAK,YAAA,CAAa,GAAA,CAAI,IAAA,CAAK,CAAC,CAAE,CAAA,IAAK,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,IAAO,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,IAAO,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,CAAA,EAAM;AAClI,MAAA,IAAI,IAAA,CAAK,CAAC,CAAA,KAAM,GAAA,eAAkB,CAAC,UAAA;AACnC,MAAA,CAAA,EAAA;AAAA,IACF;AAEA,IAAA,IAAI,KAAK,CAAA,IAAK,IAAA,CAAK,KAAK,IAAA,CAAK,CAAC,CAAE,CAAA,EAAG;AACjC,MAAA,IAAA,CAAK,OAAO,CAAC,CAAA;AACb,MAAA,KAAA,GAAQ,CAAA;AACR,MAAA,CAAA,GAAI,CAAA,GAAI,CAAA;AAAA,IACV;AAAA,EACF;AACA,EAAA,IAAI,KAAA,GAAQ,CAAA,EAAG,IAAA,CAAK,KAAA,EAAO,CAAC,CAAA;AAC5B,EAAA,OAAO,GAAA;AACT;;;ACDO,SAAS,QAAA,CAAS,OAAA,EAA2B,OAAA,GAA2B,EAAC,EAAmB;AACjG,EAAA,MAAM,WAAA,GAAc,QAAQ,WAAA,IAAe,IAAA;AAC3C,EAAA,MAAM,MAAA,GAAS,QAAQ,qBAAA,IAAyB,CAAA;AAChD,EAAA,MAAM,MAAA,GAAS,QAAQ,eAAA,IAAmB,GAAA;AAC1C,EAAA,MAAM,QAAA,GAAW,QAAQ,QAAA,IAAY,IAAA;AACrC,EAAA,MAAM,KAAA,GAAQ,QAAQ,WAAA,KAAgB,CAAC,MAAc,WAAA,CAAY,CAAA,EAAG,QAAQ,KAAK,CAAA,CAAA;AACjF,EAAA,MAAM,GAAA,GAAM,OAAA,CAAQ,SAAA,IAAa,OAAA,CAAQ,SAAA,CAAU,SAAS,IAAI,GAAA,CAAI,OAAA,CAAQ,SAAS,CAAA,GAAI,IAAA;AACzF,EAAA,MAAM,MAAA,GACJ,OAAA,CAAQ,MAAA,KAAW,CAAC,CAAA,KAAsC,CAAA,EAAA,EAAK,CAAA,CAAE,GAAA,GAAM,CAAC,CAAA,CAAA,EAAI,CAAA,CAAE,KAAK,CAAA,CAAA,CAAA,CAAA;AAErF,EAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,GAAA,CAAI,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,EAAO,IAAA,EAAK,IAAK,CAAA,CAAA,EAAI,CAAA,GAAI,CAAC,CAAA,CAAE,CAAA;AAGnE,EAAA,MAAM,QAAqB,EAAC;AAC5B,EAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,EAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,GAAA,EAAK,IAAA,KAAS;AAC7B,IAAA,MAAM,KAAA,GAAQ,cAAA,CAAe,GAAA,CAAI,IAAA,IAAQ,EAAE,CAAA;AAC3C,IAAA,KAAA,CAAM,OAAA,CAAQ,CAAC,GAAA,EAAK,QAAA,KAAa;AAC/B,MAAA,KAAA,CAAM,IAAA,CAAK;AAAA,QACT,MAAM,GAAA,CAAI,IAAA;AAAA,QACV,SAAA,EAAW,IAAA;AAAA,QACX,OAAO,GAAA,CAAI,KAAA;AAAA,QACX,KAAK,GAAA,CAAI,GAAA;AAAA,QACT,KAAA,EAAO,KAAA,EAAA;AAAA,QACP,KAAA,EAAO,CAAA;AAAA,QACP,MAAM,QAAA,KAAa,CAAA;AAAA,QACnB,MAAA,EAAQ,SAAA,CAAU,GAAA,CAAI,IAAI,CAAA;AAAA,QAC1B,KAAA,EAAO,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA;AAAA,QACxB,MAAA,EAAQ,SAAA,CAAU,GAAA,CAAI,IAAA,EAAM,GAAG,CAAA;AAAA,QAC/B,OAAA,EAAS,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA;AAAA,QAC1B,MAAA,EAAQ,KAAA,CAAM,GAAA,CAAI,IAAI;AAAA,OACvB,CAAA;AAAA,IACH,CAAC,CAAA;AAAA,EACH,CAAC,CAAA;AACD,EAAA,MAAM,iBAAiB,KAAA,CAAM,MAAA;AAC7B,EAAA,IAAI,mBAAmB,CAAA,EAAG;AACxB,IAAA,OAAO,EAAE,IAAA,EAAM,EAAA,EAAI,OAAA,EAAS,EAAC,EAAG,SAAA,EAAW,EAAC,EAAG,MAAA,EAAQ,CAAA,EAAG,UAAA,EAAY,CAAA,EAAG,gBAAgB,CAAA,EAAE;AAAA,EAC7F;AAGA,EAAA,MAAM,YAAA,GAAe,MAAM,GAAA,CAAI,CAAC,MAAM,CAAA,CAAE,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAA;AACtD,EAAA,MAAM,MAAA,GAAS,IAAA;AAAA,IACb,YAAA;AAAA,IACA,KAAA,CAAM,GAAA,CAAI,CAAC,CAAA,MAAO,EAAE,EAAA,EAAI,MAAA,CAAO,CAAA,CAAE,KAAK,CAAA,EAAG,IAAA,EAAM,CAAA,CAAE,MAAK,CAAE;AAAA,GAC1D;AACA,EAAA,MAAM,SAAA,GAAY,IAAI,GAAA,CAAI,MAAA,CAAO,GAAA,CAAI,CAAC,CAAA,KAAM,CAAC,CAAA,CAAE,EAAA,EAAI,CAAA,CAAE,WAAW,CAAC,CAAC,CAAA;AAClE,EAAA,KAAA,MAAW,CAAA,IAAK,KAAA,EAAO,CAAA,CAAE,KAAA,GAAQ,SAAA,CAAU,IAAI,MAAA,CAAO,CAAA,CAAE,KAAK,CAAC,CAAA,IAAK,CAAA;AAInE,EAAA,MAAM,WAAW,CAAC,CAAA,KAAA,CACf,YAAY,CAAA,CAAE,IAAA,GAAO,IAAI,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,CAAA,GAAI,MAAM,CAAA,CAAE,MAAA,GAAS,IAAI,CAAA,CAAA,IAAM,CAAA,CAAE,SAAS,CAAA,GAAI,CAAA,CAAA;AAC1F,EAAA,MAAM,MAAA,GAAS,CAAC,GAAG,KAAK,EAAE,IAAA,CAAK,CAAC,GAAG,CAAA,KAAM;AACvC,IAAA,MAAM,EAAA,GAAK,SAAS,CAAC,CAAA;AACrB,IAAA,MAAM,EAAA,GAAK,SAAS,CAAC,CAAA;AACrB,IAAA,IAAI,EAAA,KAAO,EAAA,EAAI,OAAO,EAAA,GAAK,EAAA;AAC3B,IAAA,IAAI,EAAE,KAAA,KAAU,CAAA,CAAE,OAAO,OAAO,CAAA,CAAE,QAAQ,CAAA,CAAE,KAAA;AAC5C,IAAA,OAAO,CAAA,CAAE,QAAQ,CAAA,CAAE,KAAA;AAAA,EACrB,CAAC,CAAA;AAGD,EAAA,MAAM,OAAoB,EAAC;AAC3B,EAAA,MAAM,eAA8B,EAAC;AACrC,EAAA,MAAM,SAAA,uBAAgB,GAAA,EAAoB;AAC1C,EAAA,IAAI,IAAA,GAAO,CAAA;AACX,EAAA,IAAI,UAAA,GAAa,CAAA;AAEjB,EAAA,KAAA,MAAW,KAAK,MAAA,EAAQ;AACtB,IAAA,IAAI,GAAA,GAAM,KAAA;AACV,IAAA,KAAA,MAAW,MAAM,YAAA,EAAc;AAC7B,MAAA,IAAI,OAAA,CAAQ,CAAA,CAAE,OAAA,EAAS,EAAE,KAAK,MAAA,EAAQ;AACpC,QAAA,GAAA,GAAM,IAAA;AACN,QAAA;AAAA,MACF;AAAA,IACF;AACA,IAAA,IAAI,GAAA,EAAK;AACP,MAAA,UAAA,EAAA;AACA,MAAA;AAAA,IACF;AACA,IAAA,IAAA,CAAK,UAAU,GAAA,CAAI,CAAA,CAAE,SAAS,CAAA,IAAK,MAAM,MAAA,EAAQ;AACjD,IAAA,IAAI,OAAO,CAAA,CAAE,MAAA,GAAS,WAAA,IAAe,IAAA,CAAK,SAAS,CAAA,EAAG;AACtD,IAAA,IAAA,CAAK,KAAK,CAAC,CAAA;AACX,IAAA,YAAA,CAAa,IAAA,CAAK,EAAE,OAAO,CAAA;AAC3B,IAAA,SAAA,CAAU,GAAA,CAAI,EAAE,SAAA,EAAA,CAAY,SAAA,CAAU,IAAI,CAAA,CAAE,SAAS,CAAA,IAAK,CAAA,IAAK,CAAC,CAAA;AAChE,IAAA,IAAA,IAAQ,CAAA,CAAE,MAAA;AAAA,EACZ;AAGA,EAAA,MAAM,QAAA,uBAAe,GAAA,EAAyB;AAC9C,EAAA,KAAA,MAAW,KAAK,IAAA,EAAM;AACpB,IAAA,MAAM,MAAM,QAAA,CAAS,GAAA,CAAI,CAAA,CAAE,SAAS,KAAK,EAAC;AAC1C,IAAA,GAAA,CAAI,KAAK,CAAC,CAAA;AACV,IAAA,QAAA,CAAS,GAAA,CAAI,CAAA,CAAE,SAAA,EAAW,GAAG,CAAA;AAAA,EAC/B;AAEA,EAAA,MAAM,KAAA,GAAQ,CAAC,CAAA,KAAoC;AACjD,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,IAAI,QAAA,IAAY,CAAA,CAAE,IAAA,EAAM,OAAA,CAAQ,KAAK,MAAM,CAAA;AAC3C,IAAA,IAAI,CAAA,CAAE,KAAA,EAAO,OAAA,CAAQ,IAAA,CAAK,OAAO,CAAA;AACjC,IAAA,IAAI,CAAA,CAAE,MAAA,EAAQ,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AACnC,IAAA,IAAI,CAAA,CAAE,MAAA,EAAQ,OAAA,CAAQ,IAAA,CAAK,QAAQ,CAAA;AACnC,IAAA,IAAI,OAAA,CAAQ,MAAA,KAAW,CAAA,EAAG,OAAA,CAAQ,KAAK,MAAM,CAAA;AAC7C,IAAA,OAAO,EAAE,IAAA,EAAM,CAAA,CAAE,IAAA,EAAM,SAAA,EAAW,EAAE,SAAA,EAAW,KAAA,EAAO,CAAA,CAAE,KAAA,EAAO,KAAK,CAAA,CAAE,GAAA,EAAK,KAAA,EAAO,CAAA,CAAE,OAAO,OAAA,EAAQ;AAAA,EACrG,CAAA;AAEA,EAAA,MAAM,aAAgC,EAAC;AACvC,EAAA,MAAM,OAA4B,EAAC;AACnC,EAAA,MAAM,QAAkB,EAAC;AACzB,EAAA,OAAA,CAAQ,OAAA,CAAQ,CAAC,GAAA,EAAK,IAAA,KAAS;AAC7B,IAAA,MAAM,IAAA,GAAA,CAAQ,QAAA,CAAS,GAAA,CAAI,IAAI,KAAK,EAAC,EAAG,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,EAAE,KAAK,CAAA;AACxE,IAAA,IAAI,IAAA,CAAK,WAAW,CAAA,EAAG;AACvB,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,GAAA,CAAI,KAAK,CAAA;AAC/B,IAAA,UAAA,CAAW,IAAA,CAAK,EAAE,GAAA,EAAK,IAAA,EAAM,KAAA,EAAO,MAAA,CAAO,IAAI,CAAA,EAAI,GAAA,EAAK,GAAA,CAAI,GAAA,EAAK,SAAA,EAAW,UAAU,CAAA;AACtF,IAAA,IAAA,CAAK,IAAA,CAAK,GAAG,QAAQ,CAAA;AACrB,IAAA,KAAA,CAAM,IAAA,CAAK,CAAA,EAAG,MAAA,CAAO,EAAE,KAAK,IAAA,EAAM,KAAA,EAAO,MAAA,CAAO,IAAI,CAAA,EAAI,GAAA,EAAK,GAAA,CAAI,GAAA,EAAK,CAAC;AAAA,EAAK,QAAA,CAAS,GAAA,CAAI,CAAC,CAAA,KAAM,CAAA,CAAE,IAAI,CAAA,CAAE,IAAA,CAAK,GAAG,CAAC,CAAA,CAAE,CAAA;AAAA,EACrH,CAAC,CAAA;AAED,EAAA,MAAM,IAAA,GAAO,KAAA,CAAM,IAAA,CAAK,MAAM,CAAA;AAC9B,EAAA,OAAO,EAAE,IAAA,EAAM,OAAA,EAAS,UAAA,EAAY,SAAA,EAAW,IAAA,EAAM,MAAA,EAAQ,KAAA,CAAM,IAAI,CAAA,EAAG,UAAA,EAAY,cAAA,EAAe;AACvG","file":"index.js","sourcesContent":["// Zero-dep detectors for the sentence features we always want to keep.\n\nconst CURRENCY = /(?:रु|नेरु|रुपैयाँ|₹|\\$|€|£|Rs\\.?|USD|NPR|INR)/i;\n// A number: Latin or Devanagari digits, optionally with separators/percent.\nconst NUMBERISH = /[0-9०-९][0-9०-९.,%]*/;\nconst PERCENT = /[%]|प्रतिशत/;\n// A quotation: matched straight or curly quotes with content between.\nconst QUOTED = /\"[^\"]{3,}\"|“[^”]{3,}”|‘[^’]{3,}’/;\n\nexport function hasNumber(text: string): boolean {\n  return NUMBERISH.test(text) || CURRENCY.test(text) || PERCENT.test(text);\n}\n\nexport function hasQuote(text: string): boolean {\n  return QUOTED.test(text);\n}\n\n/** True if any gazetteer term appears in the text (case-insensitive, whole-token where Latin). */\nexport function hasEntity(text: string, gaz: Set<string> | null): boolean {\n  if (!gaz || gaz.size === 0) return false;\n  const lower = text.toLowerCase();\n  for (const term of gaz) {\n    if (!term) continue;\n    // Devanagari has no case; Latin terms are matched with word boundaries.\n    if (/[^\\u0000-\\u007F]/.test(term)) {\n      if (text.includes(term)) return true;\n    } else if (new RegExp(`(?:^|[^a-z])${escapeRe(term.toLowerCase())}(?:[^a-z]|$)`).test(lower)) {\n      return true;\n    }\n  }\n  return false;\n}\n\nfunction escapeRe(s: string): string {\n  return s.replace(/[.*+?^${}()|[\\]\\\\]/g, \"\\\\$&\");\n}\n\n/** Word 3-gram shingles for near-duplicate detection. */\nexport function shingles(text: string, size = 3): Set<string> {\n  const words = text\n    .toLowerCase()\n    .replace(/[^\\p{L}\\p{N}\\s]/gu, \" \")\n    .split(/\\s+/)\n    .filter(Boolean);\n  const out = new Set<string>();\n  if (words.length < size) {\n    if (words.length) out.add(words.join(\" \"));\n    return out;\n  }\n  for (let i = 0; i + size <= words.length; i++) out.add(words.slice(i, i + size).join(\" \"));\n  return out;\n}\n\nexport function jaccard(a: Set<string>, b: Set<string>): number {\n  if (a.size === 0 && b.size === 0) return 0;\n  let inter = 0;\n  const [small, large] = a.size <= b.size ? [a, b] : [b, a];\n  for (const t of small) if (large.has(t)) inter++;\n  return inter / (a.size + b.size - inter);\n}\n","/** A sentence with its character offsets in the original source text. */\nexport interface Sentence {\n  text: string;\n  /** Inclusive start offset in the source. */\n  start: number;\n  /** Exclusive end offset in the source. */\n  end: number;\n}\n\n// Terminators: Latin . ! ? plus Devanagari danda । and double danda ॥ and the ellipsis.\nconst TERMINATORS = new Set([\".\", \"!\", \"?\", \"।\", \"॥\", \"…\"]);\n// Quote characters whose parity we track so we never split inside a quote.\nconst OPEN_QUOTES = new Set([\"“\", \"‘\", \"«\"]); // \" ' «\nconst CLOSE_QUOTES = new Set([\"”\", \"’\", \"»\"]); // \" ' »\n\n// Common abbreviations after which a period does NOT end a sentence.\nconst ABBREV = new Set([\n  \"mr\", \"mrs\", \"ms\", \"dr\", \"prof\", \"sr\", \"jr\", \"st\", \"vs\", \"etc\", \"inc\", \"ltd\",\n  \"co\", \"corp\", \"govt\", \"gen\", \"rep\", \"sen\", \"gov\", \"no\", \"vol\", \"fig\", \"al\",\n  \"rs\", \"u.s\", \"u.k\", \"e.g\", \"i.e\", \"a.m\", \"p.m\",\n]);\n\nfunction isDigit(ch: string): boolean {\n  return (ch >= \"0\" && ch <= \"9\") || (ch >= \"०\" && ch <= \"९\");\n}\n\n/**\n * Split `text` into sentences with character offsets. Handles Latin and\n * Devanagari terminators (। ॥), never splits inside a quotation, and does not\n * break on decimals (3.5), abbreviations (Dr.) or a terminator glued to a digit.\n * Deterministic: identical input always yields identical output.\n */\nexport function splitSentences(text: string): Sentence[] {\n  const out: Sentence[] = [];\n  if (!text) return out;\n  let quoteDepth = 0;\n  let doubleOpen = false;\n  let start = 0;\n  const n = text.length;\n\n  const push = (from: number, to: number) => {\n    const raw = text.slice(from, to);\n    const trimmedStart = raw.length - raw.trimStart().length;\n    const trimmedEnd = raw.length - raw.trimEnd().length;\n    const s = from + trimmedStart;\n    const e = to - trimmedEnd;\n    if (e > s) out.push({ text: text.slice(s, e), start: s, end: e });\n  };\n\n  for (let i = 0; i < n; i++) {\n    const ch = text[i]!;\n    if (ch === '\"') {\n      doubleOpen = !doubleOpen;\n      continue;\n    }\n    if (OPEN_QUOTES.has(ch)) {\n      quoteDepth++;\n      continue;\n    }\n    if (CLOSE_QUOTES.has(ch)) {\n      if (quoteDepth > 0) quoteDepth--;\n      continue;\n    }\n    if (!TERMINATORS.has(ch)) continue;\n    if (quoteDepth > 0 || doubleOpen) continue; // inside a quote — keep it whole\n\n    // A period between digits (3.5) or in an abbreviation is not a break.\n    if (ch === \".\") {\n      const prev = text[i - 1];\n      const next = text[i + 1];\n      if (prev && next && isDigit(prev) && isDigit(next)) continue;\n      // trailing abbreviation like \"Dr.\" — look back to the word\n      let j = i - 1;\n      while (j >= 0 && /[A-Za-z.]/.test(text[j]!)) j--;\n      const word = text.slice(j + 1, i).toLowerCase();\n      if (ABBREV.has(word)) continue;\n    }\n\n    // Consume any run of terminators/closing quotes/brackets.\n    let k = i + 1;\n    while (k < n && (TERMINATORS.has(text[k]!) || CLOSE_QUOTES.has(text[k]!) || text[k] === '\"' || text[k] === \")\" || text[k] === \"]\")) {\n      if (text[k] === '\"') doubleOpen = !doubleOpen;\n      k++;\n    }\n    // Must be followed by whitespace or end of text to count as a break.\n    if (k >= n || /\\s/.test(text[k]!)) {\n      push(start, k);\n      start = k;\n      i = k - 1;\n    }\n  }\n  if (start < n) push(start, n);\n  return out;\n}\n","import { bm25 } from \"@lacspace/rerank\";\nimport { countTokens } from \"@lacspace/tokenizer\";\nimport { hasEntity, hasNumber, hasQuote, jaccard, shingles } from \"./features.js\";\nimport { splitSentences } from \"./sentences.js\";\n\nexport { splitSentences } from \"./sentences.js\";\nexport type { Sentence } from \"./sentences.js\";\n\n/** One input article about the story. */\nexport interface CondenseSource {\n  text: string;\n  /** Short label used in the grouped output header, e.g. \"Kathmandu Post\". Defaults to \"S{n}\". */\n  label?: string;\n  url?: string;\n  publishedAt?: string | Date;\n}\n\nexport interface CondenseOptions {\n  /** Total token budget for the condensed digest. Default 1500. */\n  tokenBudget?: number;\n  /** Cap on kept sentences from any single source, so one long article can't crowd out the rest. Default 8. */\n  maxSentencesPerSource?: number;\n  /** Drop a sentence whose 3-gram Jaccard similarity to an already-kept sentence is ≥ this. Default 0.8. */\n  dedupeThreshold?: number;\n  /** Always keep each source's first sentence (the lede) regardless of score. Default true. */\n  keepLede?: boolean;\n  /** Named entities (people/places) to always keep and to boost — pass en and ne forms. */\n  gazetteer?: string[];\n  /** Model hint for token counting (passed to @lacspace/tokenizer). */\n  model?: string;\n  /** Override token counting entirely. Default: @lacspace/tokenizer countTokens. */\n  countTokens?: (text: string) => number;\n  /** Header format for each source group. Default `[S{n} {label}]`. */\n  header?: (source: { idx: number; label: string; url?: string }) => string;\n}\n\nexport interface CondensedSentence {\n  text: string;\n  /** Index of the source this sentence came from (0-based). */\n  sourceIdx: number;\n  /** Character offsets in that source's original text. */\n  start: number;\n  end: number;\n  /** BM25 centrality score within the cluster. */\n  score: number;\n  /** Why it was kept: any of \"lede\", \"number\", \"quote\", \"entity\", \"rank\". */\n  reasons: string[];\n}\n\nexport interface CondensedSource {\n  idx: number;\n  label: string;\n  url?: string;\n  sentences: CondensedSentence[];\n}\n\nexport interface CondenseResult {\n  /** The digest: kept sentences grouped per source (source order) under a header line. */\n  text: string;\n  /** Kept sentences grouped per source, each in the source's original order. */\n  sources: CondensedSource[];\n  /** All kept sentences, flat, in output (source-grouped) order. */\n  sentences: CondensedSentence[];\n  /** Token count of `text` (same counter used for the budget). */\n  tokens: number;\n  /** How many near-duplicate sentences were dropped. */\n  droppedDup: number;\n  /** Total sentences seen across all sources. */\n  totalSentences: number;\n}\n\ninterface Candidate {\n  text: string;\n  sourceIdx: number;\n  start: number;\n  end: number;\n  order: number; // global index for stable ties\n  score: number;\n  lede: boolean;\n  number: boolean;\n  quote: boolean;\n  entity: boolean;\n  shingle: Set<string>;\n  tokens: number;\n}\n\n/**\n * Condense several sources on the same story into one short, deduplicated,\n * token-budgeted digest that keeps the numbers, quotes and named entities — so\n * an LLM only rewrites a fraction of the words. Purely extractive and\n * deterministic; no network, no model.\n */\nexport function condense(sources: CondenseSource[], options: CondenseOptions = {}): CondenseResult {\n  const tokenBudget = options.tokenBudget ?? 1500;\n  const maxPer = options.maxSentencesPerSource ?? 8;\n  const dedupe = options.dedupeThreshold ?? 0.8;\n  const keepLede = options.keepLede ?? true;\n  const count = options.countTokens ?? ((t: string) => countTokens(t, options.model));\n  const gaz = options.gazetteer && options.gazetteer.length ? new Set(options.gazetteer) : null;\n  const header =\n    options.header ?? ((s: { idx: number; label: string }) => `[S${s.idx + 1} ${s.label}]`);\n\n  const labels = sources.map((s, i) => s.label?.trim() || `S${i + 1}`);\n\n  // 1. Split every source into sentences with offsets.\n  const cands: Candidate[] = [];\n  let order = 0;\n  sources.forEach((src, sIdx) => {\n    const sents = splitSentences(src.text ?? \"\");\n    sents.forEach((sen, localIdx) => {\n      cands.push({\n        text: sen.text,\n        sourceIdx: sIdx,\n        start: sen.start,\n        end: sen.end,\n        order: order++,\n        score: 0,\n        lede: localIdx === 0,\n        number: hasNumber(sen.text),\n        quote: hasQuote(sen.text),\n        entity: hasEntity(sen.text, gaz),\n        shingle: shingles(sen.text),\n        tokens: count(sen.text),\n      });\n    });\n  });\n  const totalSentences = cands.length;\n  if (totalSentences === 0) {\n    return { text: \"\", sources: [], sentences: [], tokens: 0, droppedDup: 0, totalSentences: 0 };\n  }\n\n  // 2. Score sentence centrality with BM25 against the whole cluster.\n  const clusterQuery = cands.map((c) => c.text).join(\" \");\n  const scored = bm25(\n    clusterQuery,\n    cands.map((c) => ({ id: String(c.order), text: c.text })),\n  );\n  const scoreById = new Map(scored.map((s) => [s.id, s.rerankScore]));\n  for (const c of cands) c.score = scoreById.get(String(c.order)) ?? 0;\n\n  // 3. Priority: always-keep (lede/number/quote/entity) first, then by score.\n  //    Deterministic tie-break by global order.\n  const priority = (c: Candidate) =>\n    (keepLede && c.lede ? 4 : 0) + (c.quote ? 3 : 0) + (c.number ? 2 : 0) + (c.entity ? 1 : 0);\n  const ranked = [...cands].sort((a, b) => {\n    const pa = priority(a);\n    const pb = priority(b);\n    if (pa !== pb) return pb - pa;\n    if (b.score !== a.score) return b.score - a.score;\n    return a.order - b.order;\n  });\n\n  // 4. Greedy selection: dedupe against kept, respect per-source cap and budget.\n  const kept: Candidate[] = [];\n  const keptShingles: Set<string>[] = [];\n  const perSource = new Map<number, number>();\n  let used = 0;\n  let droppedDup = 0;\n\n  for (const c of ranked) {\n    let dup = false;\n    for (const ks of keptShingles) {\n      if (jaccard(c.shingle, ks) >= dedupe) {\n        dup = true;\n        break;\n      }\n    }\n    if (dup) {\n      droppedDup++;\n      continue;\n    }\n    if ((perSource.get(c.sourceIdx) ?? 0) >= maxPer) continue;\n    if (used + c.tokens > tokenBudget && kept.length > 0) continue; // keep at least one\n    kept.push(c);\n    keptShingles.push(c.shingle);\n    perSource.set(c.sourceIdx, (perSource.get(c.sourceIdx) ?? 0) + 1);\n    used += c.tokens;\n  }\n\n  // 5. Group kept sentences per source, each in original reading order.\n  const bySource = new Map<number, Candidate[]>();\n  for (const c of kept) {\n    const arr = bySource.get(c.sourceIdx) ?? [];\n    arr.push(c);\n    bySource.set(c.sourceIdx, arr);\n  }\n\n  const toOut = (c: Candidate): CondensedSentence => {\n    const reasons: string[] = [];\n    if (keepLede && c.lede) reasons.push(\"lede\");\n    if (c.quote) reasons.push(\"quote\");\n    if (c.number) reasons.push(\"number\");\n    if (c.entity) reasons.push(\"entity\");\n    if (reasons.length === 0) reasons.push(\"rank\");\n    return { text: c.text, sourceIdx: c.sourceIdx, start: c.start, end: c.end, score: c.score, reasons };\n  };\n\n  const outSources: CondensedSource[] = [];\n  const flat: CondensedSentence[] = [];\n  const parts: string[] = [];\n  sources.forEach((src, sIdx) => {\n    const list = (bySource.get(sIdx) ?? []).sort((a, b) => a.start - b.start);\n    if (list.length === 0) return;\n    const outSents = list.map(toOut);\n    outSources.push({ idx: sIdx, label: labels[sIdx]!, url: src.url, sentences: outSents });\n    flat.push(...outSents);\n    parts.push(`${header({ idx: sIdx, label: labels[sIdx]!, url: src.url })}\\n${outSents.map((s) => s.text).join(\" \")}`);\n  });\n\n  const text = parts.join(\"\\n\\n\");\n  return { text, sources: outSources, sentences: flat, tokens: count(text), droppedDup, totalSentences };\n}\n"]}

package/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@lacspace/condense",
+  "version": "1.0.0",
+  "description": "Extractive multi-source condenser — turn several articles on the same story into one short, deduplicated, token-budgeted digest that keeps the numbers, quotes and named entities, so an LLM only has to rewrite a fraction of the text. BM25 sentence ranking, near-duplicate removal, Devanagari-aware. Zero external deps, isomorphic, deterministic.",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": { "types": "./dist/index.d.ts", "default": "./dist/index.js" },
+      "require": { "types": "./dist/index.d.cts", "default": "./dist/index.cjs" }
+    }
+  },
+  "files": ["dist"],
+  "sideEffects": false,
+  "scripts": { "build": "tsup", "prepublishOnly": "npm run build" },
+  "keywords": ["condense", "summarize", "extractive-summary", "multi-document", "bm25", "token-budget", "dedupe", "llm", "rag", "prompt-compression", "devanagari", "nepali", "zero-dependency", "isomorphic", "typescript"],
+  "dependencies": { "@lacspace/rerank": "^1.0.0", "@lacspace/tokenizer": "^1.1.0" },
+  "author": "Lacspace <contact@lacspace.com>",
+  "license": "SEE LICENSE IN LICENSE",
+  "homepage": "https://developer.lacspace.com/packages/condense",
+  "repository": { "type": "git", "url": "git+https://github.com/lacspace/npm-packages.git", "directory": "condense" },
+  "bugs": { "url": "https://github.com/lacspace/npm-packages/issues" },
+  "engines": { "node": ">=18" },
+  "publishConfig": { "access": "public" }
+}