@anuma/sdk 10.6.1 → 10.6.2

package/dist/pii/transformers.cjs

@@ -42,20 +42,24 @@ var DEFAULT_TAG_MAP = {
   MISC: void 0
 };
 function placeTokens(tokens, text) {
-  const placed = [];
+  return tokens.map((token) => {
+    const { start = -1, end = -1 } = token;
+    return Number.isInteger(start) && Number.isInteger(end) && start >= 0 && end > start && end <= text.length ? { ...token, start, end } : { ...token, start: -1, end: -1 };
+  });
+}
+function tokenOffsets(text, tokenizer) {
+  const { input_ids } = tokenizer(text, { return_tensor: false, truncation: true });
   let cursor = 0;
-  for (const t of tokens) {
-    const surface = t.word.startsWith("##") ? t.word.slice(2) : t.word;
+  return input_ids.map((id) => {
+    const word = tokenizer.decode([id], { skip_special_tokens: true });
+    const surface = word.startsWith("##") ? word.slice(2) : word;
+    if (!surface) return { start: -1, end: -1 };
     const start = text.indexOf(surface, cursor);
-    if (start < 0) {
-      placed.push({ ...t, start: -1, end: -1 });
-      continue;
-    }
+    if (start < 0) return { start: -1, end: -1 };
     const end = start + surface.length;
-    placed.push({ ...t, start, end });
     cursor = end;
-  }
-  return placed;
+    return { start, end };
+  });
 }
 function aggregateTokens(tokens, text, tagMap, minScore) {
   const placed = placeTokens(tokens, text);
@@ -115,7 +119,14 @@ function createTransformersNerDetector(options = {}) {
       if (!text) return [];
       const classify = await load();
       const tokens = await classify(text);
-      return aggregateTokens(tokens, text, tagMap, minScore);
+      if (!tokens.length) return [];
+      const offsets = tokenOffsets(text, classify.tokenizer);
+      return aggregateTokens(
+        tokens.map((token) => ({ ...token, ...offsets[token.index] })),
+        text,
+        tagMap,
+        minScore
+      );
     }
   };
 }

package/dist/pii/transformers.d.mts

@@ -54,13 +54,10 @@ interface RawToken {
     score: number;
     index: number;
     word: string;
+    start?: number;
+    end?: number;
 }
-/**
- * Aggregate B-/I- subword tokens into whole-entity spans. A new entity starts
- * on a `B-` tag or a tag change; contiguous `I-` tokens (and WordPiece "##"
- * continuations) of the same base tag extend it. (The SDK additionally snaps
- * spans to word boundaries, so dropped trailing subwords are recovered there.)
- */
+/** Aggregate classified tokens with character offsets into whole-entity spans. */
 declare function aggregateTokens(tokens: RawToken[], text: string, tagMap: Record<string, string | undefined>, minScore: number): PiiSpan[];
 /**
  * Create a Transformers.js-backed {@link NerDetector}. The returned detector

package/dist/pii/transformers.d.ts

@@ -54,13 +54,10 @@ interface RawToken {
     score: number;
     index: number;
     word: string;
+    start?: number;
+    end?: number;
 }
-/**
- * Aggregate B-/I- subword tokens into whole-entity spans. A new entity starts
- * on a `B-` tag or a tag change; contiguous `I-` tokens (and WordPiece "##"
- * continuations) of the same base tag extend it. (The SDK additionally snaps
- * spans to word boundaries, so dropped trailing subwords are recovered there.)
- */
+/** Aggregate classified tokens with character offsets into whole-entity spans. */
 declare function aggregateTokens(tokens: RawToken[], text: string, tagMap: Record<string, string | undefined>, minScore: number): PiiSpan[];
 /**
  * Create a Transformers.js-backed {@link NerDetector}. The returned detector

package/dist/pii/transformers.mjs

@@ -7,20 +7,24 @@ var DEFAULT_TAG_MAP = {
   MISC: void 0
 };
 function placeTokens(tokens, text) {
-  const placed = [];
+  return tokens.map((token) => {
+    const { start = -1, end = -1 } = token;
+    return Number.isInteger(start) && Number.isInteger(end) && start >= 0 && end > start && end <= text.length ? { ...token, start, end } : { ...token, start: -1, end: -1 };
+  });
+}
+function tokenOffsets(text, tokenizer) {
+  const { input_ids } = tokenizer(text, { return_tensor: false, truncation: true });
   let cursor = 0;
-  for (const t of tokens) {
-    const surface = t.word.startsWith("##") ? t.word.slice(2) : t.word;
+  return input_ids.map((id) => {
+    const word = tokenizer.decode([id], { skip_special_tokens: true });
+    const surface = word.startsWith("##") ? word.slice(2) : word;
+    if (!surface) return { start: -1, end: -1 };
     const start = text.indexOf(surface, cursor);
-    if (start < 0) {
-      placed.push({ ...t, start: -1, end: -1 });
-      continue;
-    }
+    if (start < 0) return { start: -1, end: -1 };
     const end = start + surface.length;
-    placed.push({ ...t, start, end });
     cursor = end;
-  }
-  return placed;
+    return { start, end };
+  });
 }
 function aggregateTokens(tokens, text, tagMap, minScore) {
   const placed = placeTokens(tokens, text);
@@ -80,7 +84,14 @@ function createTransformersNerDetector(options = {}) {
       if (!text) return [];
       const classify = await load();
       const tokens = await classify(text);
-      return aggregateTokens(tokens, text, tagMap, minScore);
+      if (!tokens.length) return [];
+      const offsets = tokenOffsets(text, classify.tokenizer);
+      return aggregateTokens(
+        tokens.map((token) => ({ ...token, ...offsets[token.index] })),
+        text,
+        tagMap,
+        minScore
+      );
     }
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@anuma/sdk",
-  "version": "10.6.1",
+  "version": "10.6.2",
   "description": "",
   "main": "./dist/index.cjs",
   "module": "./dist/index.mjs",