@nationaldesignstudio/rampart 0.1.1 → 0.1.2

package/dist/src/ner/classifier.d.ts

@@ -21,6 +21,8 @@ interface RawEntity {
     readonly start: number;
     readonly end: number;
     readonly word: string;
+    /** Index into `[CLS] + content + [SEP]`; used for tokenizer-backed offset recovery. */
+    readonly index?: number;
 }
 /** Counts the model tokens in a string, excluding the [CLS]/[SEP] specials. */
 export type TokenCounter = (text: string) => number;
@@ -36,6 +38,8 @@ export interface TokenClassifier {
         aggregation_strategy?: "simple" | "first" | "max";
     }): Promise<RawEntity[]>;
     countTokens?: TokenCounter;
+    /** WordPiece tokens for `text` (no specials); drives index→char offset recovery. */
+    tokenize?: (text: string) => readonly string[];
 }
 /** The shipped Rampart token-classifier on Hugging Face (q4 ONNX only). */
 export declare const RAMPART_MODEL_ID = "nationaldesignstudio/rampart";

package/dist/src/ner/classifier.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../../src/ner/classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAY,IAAI,EAAE,MAAM,UAAU,CAAC;AAE/C,0EAA0E;AAC1E,UAAU,SAAS;IACjB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,oBAAoB,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,KAAK,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACtG,WAAW,CAAC,EAAE,YAAY,CAAC;CAC5B;AA6BD,2EAA2E;AAC3E,eAAO,MAAM,gBAAgB,iCAAiC,CAAC;AAE/D,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IAC5C,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAiBD,wFAAwF;AACxF,eAAO,MAAM,gBAAgB,QAAyC,CAAC;AAEvE;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAWpC;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CA0B1F;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,eAAe,EAC3B,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,IAAI,EAAE,CAAC,CAoBjB"}
+{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../../src/ner/classifier.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAGH,OAAO,KAAK,EAAY,IAAI,EAAE,MAAM,UAAU,CAAC;AAE/C,0EAA0E;AAC1E,UAAU,SAAS;IACjB,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,uFAAuF;IACvF,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,+EAA+E;AAC/E,MAAM,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,oBAAoB,CAAC,EAAE,QAAQ,GAAG,OAAO,GAAG,KAAK,CAAA;KAAE,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACtG,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,oFAAoF;IACpF,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,SAAS,MAAM,EAAE,CAAC;CAChD;AA6BD,2EAA2E;AAC3E,eAAO,MAAM,gBAAgB,iCAAiC,CAAC;AAE/D,MAAM,WAAW,UAAU;IACzB;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,wEAAwE;IACxE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;IAC5C,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAiBD,wFAAwF;AACxF,eAAO,MAAM,gBAAgB,QAAyC,CAAC;AAEvE;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,KAAK,CAAC;AAWpC;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CAgC1F;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,eAAe,EAC3B,QAAQ,GAAE,MAAiC,GAC1C,OAAO,CAAC,IAAI,EAAE,CAAC,CAoBjB"}

package/dist/src/ner/worker.js

@@ -34966,6 +34966,9 @@ async function loadNerClassifier(options = {}) {
   if (tokenizer?.encode) {
     adapter.countTokens = (text) => tokenizer.encode(text, { add_special_tokens: false }).length;
   }
+  if (tokenizer?.tokenize) {
+    adapter.tokenize = (text) => tokenizer.tokenize(text);
+  }
   return adapter;
 }
 async function detectNer(raw, classifier, minScore = DEFAULT_OPTIONS.minScore) {
@@ -35057,8 +35060,9 @@ async function detectNerWindow(raw, classifier, minScore = DEFAULT_OPTIONS.minSc
   const inferText = raw.replaceAll("-", " ");
   const entities = await classifier(inferText, { aggregation_strategy: "simple" });
   const folded = foldForModel(inferText);
+  const indexOffsets = classifier.tokenize === undefined ? undefined : buildTokenIndexOffsets(folded, classifier.tokenize(inferText));
   const candidates = [];
-  for (const entity of mergeBioTokens(entities, folded)) {
+  for (const entity of mergeBioTokens(entities, folded, indexOffsets)) {
     const label = GROUP_TO_LABEL[entity.group.toUpperCase()];
     if (label === undefined)
       continue;
@@ -35095,7 +35099,31 @@ function foldForModel(raw) {
   }
   return { text, rawStart, rawEnd };
 }
-function mergeBioTokens(entities, folded) {
+function buildTokenIndexOffsets(folded, contentTokens) {
+  const offsets = [[0, 0]];
+  let cursor = 0;
+  for (const token of contentTokens) {
+    const piece = token.startsWith("##") ? token.slice(2) : token;
+    const at = token.startsWith("##") ? cursor : folded.text.indexOf(piece, cursor);
+    if (piece.length === 0 || at < 0 || at + piece.length > folded.text.length) {
+      offsets.push([0, 0]);
+      continue;
+    }
+    offsets.push([folded.rawStart[at], folded.rawEnd[at + piece.length - 1]]);
+    cursor = at + piece.length;
+  }
+  offsets.push([0, 0]);
+  return offsets;
+}
+function offsetFromTokenIndex(indexOffsets, index) {
+  if (indexOffsets === undefined || index === undefined)
+    return null;
+  const pair = indexOffsets[index];
+  if (pair === undefined || pair[0] === pair[1])
+    return null;
+  return pair;
+}
+function mergeBioTokens(entities, folded, indexOffsets) {
   const out = [];
   let cursor = 0;
   let current = null;
@@ -35115,16 +35143,24 @@ function mergeBioTokens(entities, folded) {
     if (rawLabel === undefined)
       continue;
     const { prefix, base } = stripBio(rawLabel);
-    const word = (entity.word ?? "").replace(/^##/, "").toLowerCase();
-    if (!word)
-      continue;
-    const at = folded.text.indexOf(word, cursor);
-    if (at < 0)
-      continue;
-    const start = folded.rawStart[at];
-    const end = folded.rawEnd[at + word.length - 1];
-    cursor = at + word.length;
-    const continues = current !== null && current.group === base && prefix !== "B";
+    const indexed = offsetFromTokenIndex(indexOffsets, entity.index);
+    let start;
+    let end;
+    if (indexed !== null) {
+      [start, end] = indexed;
+    } else {
+      const word = (entity.word ?? "").replace(/^##/, "").toLowerCase();
+      if (!word)
+        continue;
+      const at = folded.text.indexOf(word, cursor);
+      if (at < 0)
+        continue;
+      start = folded.rawStart[at];
+      end = folded.rawEnd[at + word.length - 1];
+      cursor = at + word.length;
+    }
+    const isSubword = (entity.word ?? "").startsWith("##");
+    const continues = current !== null && current.group === base && (prefix !== "B" || isSubword);
     if (continues && current !== null) {
       current.end = end;
       current.score += entity.score;
@@ -35161,7 +35197,7 @@ function repairSpans(raw, spans, anchorScore) {
     }
     kept = merged;
     for (let i = 0;i < kept.length; i++) {
-      const repaired = rescueCapitalizedParticles(raw, kept[i]);
+      const repaired = rescueCapitalizedParticles(raw, kept[i], kept, i);
       if (repaired.start !== kept[i].start || repaired.end !== kept[i].end) {
         kept[i] = repaired;
         changed = true;
@@ -35208,17 +35244,28 @@ function mergeAdjacentConnectors(raw, spans) {
   }
   return merged;
 }
-function rescueCapitalizedParticles(raw, span) {
+function rescueCapitalizedParticles(raw, span, all = [], selfIndex = -1) {
   if (!PERSON_LABELS.has(span.label))
     return span;
+  let leftBound = 0;
+  let rightBound = raw.length;
+  for (let i = 0;i < all.length; i++) {
+    if (i === selfIndex)
+      continue;
+    const other = all[i];
+    if (other.end <= span.start && other.end > leftBound)
+      leftBound = other.end;
+    if (other.start >= span.end && other.start < rightBound)
+      rightBound = other.start;
+  }
   let start = span.start;
   let end = span.end;
   const left = LEFT_PARTICLE_RE.exec(raw.slice(0, start));
-  if (left !== null && CONNECTOR_RE.test(left[2]) && (!left[2].includes(".") || left[1].length === 1)) {
+  if (left !== null && CONNECTOR_RE.test(left[2]) && (!left[2].includes(".") || left[1].length === 1) && start - left[0].length >= leftBound) {
     start -= left[0].length;
   }
   const right = RIGHT_PARTICLE_RE.exec(raw.slice(end));
-  if (right !== null && !right[1].includes(".")) {
+  if (right !== null && !right[1].includes(".") && end + right[0].length <= rightBound) {
     end += right[0].length;
   }
   return start === span.start && end === span.end ? span : { ...span, start, end, text: raw.slice(start, end) };
@@ -35298,5 +35345,5 @@ export {
   createWorkerClassifier
 };
 
-//# debugId=8DAAE1F4BDB1714C64756E2164756E21
+//# debugId=474CA63CBA7E3BFF64756E2164756E21
 //# sourceMappingURL=worker.js.map