@nationaldesignstudio/rampart 0.1.1 → 0.1.2

package/dist/index.js

@@ -34966,6 +34966,9 @@ async function loadNerClassifier(options = {}) {
   if (tokenizer?.encode) {
     adapter.countTokens = (text) => tokenizer.encode(text, { add_special_tokens: false }).length;
   }
+  if (tokenizer?.tokenize) {
+    adapter.tokenize = (text) => tokenizer.tokenize(text);
+  }
   return adapter;
 }
 async function detectNer(raw, classifier, minScore = DEFAULT_OPTIONS.minScore) {
@@ -35057,8 +35060,9 @@ async function detectNerWindow(raw, classifier, minScore = DEFAULT_OPTIONS.minSc
   const inferText = raw.replaceAll("-", " ");
   const entities = await classifier(inferText, { aggregation_strategy: "simple" });
   const folded = foldForModel(inferText);
+  const indexOffsets = classifier.tokenize === undefined ? undefined : buildTokenIndexOffsets(folded, classifier.tokenize(inferText));
   const candidates = [];
-  for (const entity of mergeBioTokens(entities, folded)) {
+  for (const entity of mergeBioTokens(entities, folded, indexOffsets)) {
     const label = GROUP_TO_LABEL[entity.group.toUpperCase()];
     if (label === undefined)
       continue;
@@ -35095,7 +35099,31 @@ function foldForModel(raw) {
   }
   return { text, rawStart, rawEnd };
 }
-function mergeBioTokens(entities, folded) {
+function buildTokenIndexOffsets(folded, contentTokens) {
+  const offsets = [[0, 0]];
+  let cursor = 0;
+  for (const token of contentTokens) {
+    const piece = token.startsWith("##") ? token.slice(2) : token;
+    const at = token.startsWith("##") ? cursor : folded.text.indexOf(piece, cursor);
+    if (piece.length === 0 || at < 0 || at + piece.length > folded.text.length) {
+      offsets.push([0, 0]);
+      continue;
+    }
+    offsets.push([folded.rawStart[at], folded.rawEnd[at + piece.length - 1]]);
+    cursor = at + piece.length;
+  }
+  offsets.push([0, 0]);
+  return offsets;
+}
+function offsetFromTokenIndex(indexOffsets, index) {
+  if (indexOffsets === undefined || index === undefined)
+    return null;
+  const pair = indexOffsets[index];
+  if (pair === undefined || pair[0] === pair[1])
+    return null;
+  return pair;
+}
+function mergeBioTokens(entities, folded, indexOffsets) {
   const out = [];
   let cursor = 0;
   let current = null;
@@ -35115,16 +35143,24 @@ function mergeBioTokens(entities, folded) {
     if (rawLabel === undefined)
       continue;
     const { prefix, base } = stripBio(rawLabel);
-    const word = (entity.word ?? "").replace(/^##/, "").toLowerCase();
-    if (!word)
-      continue;
-    const at = folded.text.indexOf(word, cursor);
-    if (at < 0)
-      continue;
-    const start = folded.rawStart[at];
-    const end = folded.rawEnd[at + word.length - 1];
-    cursor = at + word.length;
-    const continues = current !== null && current.group === base && prefix !== "B";
+    const indexed = offsetFromTokenIndex(indexOffsets, entity.index);
+    let start;
+    let end;
+    if (indexed !== null) {
+      [start, end] = indexed;
+    } else {
+      const word = (entity.word ?? "").replace(/^##/, "").toLowerCase();
+      if (!word)
+        continue;
+      const at = folded.text.indexOf(word, cursor);
+      if (at < 0)
+        continue;
+      start = folded.rawStart[at];
+      end = folded.rawEnd[at + word.length - 1];
+      cursor = at + word.length;
+    }
+    const isSubword = (entity.word ?? "").startsWith("##");
+    const continues = current !== null && current.group === base && (prefix !== "B" || isSubword);
     if (continues && current !== null) {
       current.end = end;
       current.score += entity.score;
@@ -35161,7 +35197,7 @@ function repairSpans(raw, spans, anchorScore) {
     }
     kept = merged;
     for (let i = 0;i < kept.length; i++) {
-      const repaired = rescueCapitalizedParticles(raw, kept[i]);
+      const repaired = rescueCapitalizedParticles(raw, kept[i], kept, i);
       if (repaired.start !== kept[i].start || repaired.end !== kept[i].end) {
         kept[i] = repaired;
         changed = true;
@@ -35208,17 +35244,28 @@ function mergeAdjacentConnectors(raw, spans) {
   }
   return merged;
 }
-function rescueCapitalizedParticles(raw, span) {
+function rescueCapitalizedParticles(raw, span, all = [], selfIndex = -1) {
   if (!PERSON_LABELS.has(span.label))
     return span;
+  let leftBound = 0;
+  let rightBound = raw.length;
+  for (let i = 0;i < all.length; i++) {
+    if (i === selfIndex)
+      continue;
+    const other = all[i];
+    if (other.end <= span.start && other.end > leftBound)
+      leftBound = other.end;
+    if (other.start >= span.end && other.start < rightBound)
+      rightBound = other.start;
+  }
   let start = span.start;
   let end = span.end;
   const left = LEFT_PARTICLE_RE.exec(raw.slice(0, start));
-  if (left !== null && CONNECTOR_RE.test(left[2]) && (!left[2].includes(".") || left[1].length === 1)) {
+  if (left !== null && CONNECTOR_RE.test(left[2]) && (!left[2].includes(".") || left[1].length === 1) && start - left[0].length >= leftBound) {
     start -= left[0].length;
   }
   const right = RIGHT_PARTICLE_RE.exec(raw.slice(end));
-  if (right !== null && !right[1].includes(".")) {
+  if (right !== null && !right[1].includes(".") && end + right[0].length <= rightBound) {
     end += right[0].length;
   }
   return start === span.start && end === span.end ? span : { ...span, start, end, text: raw.slice(start, end) };
@@ -35635,5 +35682,5 @@ export {
   ChatGuard
 };
 
-//# debugId=6D8E27590A5F82A064756E2164756E21
+//# debugId=03EC2D3780505F3564756E2164756E21
 //# sourceMappingURL=index.js.map