@x12i/api-hub-text 0.0.0-stage → 0.1.1

package/README.md

@@ -1,3 +1,11 @@
-# Temporary Holding Version
+# @x12i/api-hub-text
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Term and entity helpers for the x12i API hub
+
+ESM package for Node.js 24 or newer. Import the public name:
+
+```js
+import * as text from '@x12i/api-hub-text';
+```
+
+The whole set is also available from [`@x12i/api-hub`](https://www.npmjs.com/package/@x12i/api-hub).

package/dist/index.d.ts

@@ -0,0 +1,27 @@
+import { ENGLISH_STOPWORDS } from './stopwords.js';
+export { ENGLISH_STOPWORDS };
+export interface ScoredKeyword {
+    term: string;
+    score: number;
+}
+export interface KeywordOptions {
+    limit?: number;
+    /** Extra terms to drop, case-insensitive. */
+    ignore?: string[];
+    minLength?: number;
+}
+/** Most salient terms of one text (`@x12i/distinctive-terms`, English stopwords removed). */
+export declare function keywords(text: string, opts?: KeywordOptions): ScoredKeyword[];
+/** For each document, the terms that set it apart from the rest of the collection. */
+export declare function distinctiveTerms(documents: string[], opts?: KeywordOptions): ScoredKeyword[][];
+export type IdentifierType = 'email' | 'url' | 'ip' | 'cve' | 'domain';
+export interface IdentifierMention {
+    type: IdentifierType;
+    value: string;
+    /** Code-unit offsets into the input text. */
+    start: number;
+    end: number;
+}
+export declare const MAX_IDENTIFIER_TEXT = 1000000;
+/** Validated identifiers with their offsets (`@x12i/text-entities`). Nothing from the input is retained. */
+export declare function findIdentifiers(text: string, types?: IdentifierType[], maxMentions?: number): IdentifierMention[];

package/dist/index.js

@@ -0,0 +1,60 @@
+import { extractTerms } from '@x12i/distinctive-terms';
+import { createEntityExtractor } from '@x12i/text-entities';
+import { ENGLISH_STOPWORDS } from "./stopwords.js";
+export { ENGLISH_STOPWORDS };
+function merge(rows, limit) {
+    const byTerm = new Map();
+    for (const row of rows) {
+        const key = row.term.toLowerCase();
+        const prev = byTerm.get(key);
+        if (prev)
+            prev.score += row.score;
+        else
+            byTerm.set(key, { term: key, score: row.score });
+    }
+    return [...byTerm.values()]
+        .map((k) => ({ term: k.term, score: Math.round(k.score * 1000) / 1000 }))
+        .sort((a, b) => b.score - a.score || a.term.localeCompare(b.term))
+        .slice(0, limit);
+}
+function options(opts, topK) {
+    return {
+        topK,
+        stopwords: ENGLISH_STOPWORDS,
+        ignore: opts.ignore ?? [],
+        minTokenLength: opts.minLength ?? 3,
+        minDf: 1,
+    };
+}
+/** Most salient terms of one text (`@x12i/distinctive-terms`, English stopwords removed). */
+export function keywords(text, opts = {}) {
+    const limit = opts.limit ?? 10;
+    const [rows = []] = extractTerms([text], options(opts, limit * 3));
+    return merge(rows, limit);
+}
+/** For each document, the terms that set it apart from the rest of the collection. */
+export function distinctiveTerms(documents, opts = {}) {
+    const limit = opts.limit ?? 5;
+    return extractTerms(documents, options(opts, limit * 3)).map((rows) => merge(rows, limit));
+}
+const TO_LIB = { email: 'EMAIL', url: 'URL', ip: 'IP_ADDRESS', cve: 'CVE_ID', domain: 'DOMAIN' };
+const FROM_LIB = Object.fromEntries(Object.entries(TO_LIB).map(([k, v]) => [v, k]));
+export const MAX_IDENTIFIER_TEXT = 1_000_000;
+/** Validated identifiers with their offsets (`@x12i/text-entities`). Nothing from the input is retained. */
+export function findIdentifiers(text, types = ['email', 'url', 'ip', 'cve'], maxMentions = 1_000) {
+    const extractor = createEntityExtractor({
+        profileId: 'validated-ascii-v1',
+        types: [...new Set(types)].map((t) => TO_LIB[t]),
+        limits: { maxInputCodeUnits: MAX_IDENTIFIER_TEXT, maxMentions },
+    });
+    const result = extractor.extract({
+        documentId: 'input',
+        revision: '1',
+        text,
+        segments: [{ id: 'body', zone: 'content', path: '/body', text, language: 'und', kind: 'prose', source: { start: 0, end: text.length } }],
+    });
+    return result.items
+        .filter((m) => FROM_LIB[m.type])
+        .map((m) => ({ type: FROM_LIB[m.type], value: m.normalized, start: m.evidence.start, end: m.evidence.end }));
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../packages/text/src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,CAAC;AAc7B,SAAS,KAAK,CAAC,IAAuC,EAAE,KAAa;IACnE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,IAAI;YAAE,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC;;YAC7B,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC;SACxE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACjE,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;AACrB,CAAC;AAED,SAAS,OAAO,CAAC,IAAoB,EAAE,IAAY;IACjD,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,iBAAiB;QAC5B,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;QACzB,cAAc,EAAE,IAAI,CAAC,SAAS,IAAI,CAAC;QACnC,KAAK,EAAE,CAAC;KACT,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,OAAuB,EAAE;IAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;IAC/B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACnE,OAAO,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,gBAAgB,CAAC,SAAmB,EAAE,OAAuB,EAAE;IAC7E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;IAC9B,OAAO,YAAY,CAAC,SAAS,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC7F,CAAC;AAID,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAW,CAAC;AAC1G,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAmC,CAAC;AAUtH,MAAM,CAAC,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAE7C,4GAA4G;AAC5G,MAAM,UAAU,eAAe,CAAC,IAAY,EAAE,QAA0B,CAAC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,EAAE,WAAW,GAAG,KAAK;IACxH,MAAM,SAAS,GAAG,qBAAqB,CAAC;QACtC,SAAS,EAAE,oBAAoB;QAC/B,KAAK,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,EAAE,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,WAAW,EAAE;KAChE,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,OAAO;QACnB,QAAQ,EAAE,GAAG;QACb,IAAI;QACJ,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;KAChI,CAAC,CAAC;IACZ,OAAO,MAAM,CAAC,KAAK;SAChB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SAC/B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAE,EAAE,KAAK,EAAE,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AAClH,CAAC","sourcesContent":["import { extractTerms } from '@x12i/distinctive-terms';\nimport { createEntityExtractor } from '@x12i/text-entities';\nimport { ENGLISH_STOPWORDS } from './stopwords.ts';\n\nexport { ENGLISH_STOPWORDS };\n\nexport interface ScoredKeyword {\n  term: string;\n  score: number;\n}\n\nexport interface KeywordOptions {\n  limit?: number;\n  /** Extra terms to drop, case-insensitive. */\n  ignore?: string[];\n  minLength?: number;\n}\n\nfunction merge(rows: { term: string; score: number }[], limit: number): ScoredKeyword[] {\n  const byTerm = new Map<string, ScoredKeyword>();\n  for (const row of rows) {\n    const key = row.term.toLowerCase();\n    const prev = byTerm.get(key);\n    if (prev) prev.score += row.score;\n    else byTerm.set(key, { term: key, score: row.score });\n  }\n  return [...byTerm.values()]\n    .map((k) => ({ term: k.term, score: Math.round(k.score * 1000) / 1000 }))\n    .sort((a, b) => b.score - a.score || a.term.localeCompare(b.term))\n    .slice(0, limit);\n}\n\nfunction options(opts: KeywordOptions, topK: number) {\n  return {\n    topK,\n    stopwords: ENGLISH_STOPWORDS,\n    ignore: opts.ignore ?? [],\n    minTokenLength: opts.minLength ?? 3,\n    minDf: 1,\n  };\n}\n\n/** Most salient terms of one text (`@x12i/distinctive-terms`, English stopwords removed). */\nexport function keywords(text: string, opts: KeywordOptions = {}): ScoredKeyword[] {\n  const limit = opts.limit ?? 10;\n  const [rows = []] = extractTerms([text], options(opts, limit * 3));\n  return merge(rows, limit);\n}\n\n/** For each document, the terms that set it apart from the rest of the collection. */\nexport function distinctiveTerms(documents: string[], opts: KeywordOptions = {}): ScoredKeyword[][] {\n  const limit = opts.limit ?? 5;\n  return extractTerms(documents, options(opts, limit * 3)).map((rows) => merge(rows, limit));\n}\n\nexport type IdentifierType = 'email' | 'url' | 'ip' | 'cve' | 'domain';\n\nconst TO_LIB = { email: 'EMAIL', url: 'URL', ip: 'IP_ADDRESS', cve: 'CVE_ID', domain: 'DOMAIN' } as const;\nconst FROM_LIB = Object.fromEntries(Object.entries(TO_LIB).map(([k, v]) => [v, k])) as Record<string, IdentifierType>;\n\nexport interface IdentifierMention {\n  type: IdentifierType;\n  value: string;\n  /** Code-unit offsets into the input text. */\n  start: number;\n  end: number;\n}\n\nexport const MAX_IDENTIFIER_TEXT = 1_000_000;\n\n/** Validated identifiers with their offsets (`@x12i/text-entities`). Nothing from the input is retained. */\nexport function findIdentifiers(text: string, types: IdentifierType[] = ['email', 'url', 'ip', 'cve'], maxMentions = 1_000): IdentifierMention[] {\n  const extractor = createEntityExtractor({\n    profileId: 'validated-ascii-v1',\n    types: [...new Set(types)].map((t) => TO_LIB[t]),\n    limits: { maxInputCodeUnits: MAX_IDENTIFIER_TEXT, maxMentions },\n  });\n  const result = extractor.extract({\n    documentId: 'input',\n    revision: '1',\n    text,\n    segments: [{ id: 'body', zone: 'content', path: '/body', text, language: 'und', kind: 'prose', source: { start: 0, end: text.length } }],\n  } as never);\n  return result.items\n    .filter((m) => FROM_LIB[m.type])\n    .map((m) => ({ type: FROM_LIB[m.type]!, value: m.normalized, start: m.evidence.start, end: m.evidence.end }));\n}\n"]}

package/dist/stopwords.d.ts

@@ -0,0 +1,2 @@
+/** Function words that never make useful keywords. English only for now. */
+export declare const ENGLISH_STOPWORDS: string[];

package/dist/stopwords.js

@@ -0,0 +1,17 @@
+/** Function words that never make useful keywords. English only for now. */
+export const ENGLISH_STOPWORDS = `
+a about above after again against all almost also although always am among an and another any anyone anything are
+around as at be became because become been before being below between both but by can cannot could did do does
+doing done down during each either else enough etc even ever every few for from further get gets got had has have
+having he her here hers herself him himself his how however i if in into is it its itself just least less let like
+made make many may me might more most much must my myself near need neither never no nor not now of off often on
+once one only onto or other others otherwise our ours ourselves out over own per perhaps please put rather really
+same see seem seemed seems several shall she should since so some something sometimes still such than that the
+their theirs them themselves then there therefore these they this those though through thus to together too
+toward towards under until up upon us use used uses using very via was we well were what whatever when whenever
+where whereas wherever whether which while who whoever whom whose why will with within without would yet you your
+yours yourself yourselves
+`
+    .split(/\s+/)
+    .filter(Boolean);
+//# sourceMappingURL=stopwords.js.map

package/dist/stopwords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopwords.js","sourceRoot":"","sources":["../../../../../packages/text/src/stopwords.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,iBAAiB,GAAG;;;;;;;;;;;;CAYhC;KACE,KAAK,CAAC,KAAK,CAAC;KACZ,MAAM,CAAC,OAAO,CAAC,CAAC","sourcesContent":["/** Function words that never make useful keywords. English only for now. */\nexport const ENGLISH_STOPWORDS = `\na about above after again against all almost also although always am among an and another any anyone anything are\naround as at be became because become been before being below between both but by can cannot could did do does\ndoing done down during each either else enough etc even ever every few for from further get gets got had has have\nhaving he her here hers herself him himself his how however i if in into is it its itself just least less let like\nmade make many may me might more most much must my myself near need neither never no nor not now of off often on\nonce one only onto or other others otherwise our ours ourselves out over own per perhaps please put rather really\nsame see seem seemed seems several shall she should since so some something sometimes still such than that the\ntheir theirs them themselves then there therefore these they this those though through thus to together too\ntoward towards under until up upon us use used uses using very via was we well were what whatever when whenever\nwhere whereas wherever whether which while who whoever whom whose why will with within without would yet you your\nyours yourself yourselves\n`\n  .split(/\\s+/)\n  .filter(Boolean);\n"]}

package/package.json

@@ -1,6 +1,37 @@
 {
   "name": "@x12i/api-hub-text",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Term and entity helpers for the x12i API hub",
+  "type": "module",
+  "engines": {
+    "node": ">=24"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/x12i/apis-hub.git",
+    "directory": "packages/text"
+  },
+  "bugs": {
+    "url": "https://github.com/x12i/apis-hub/issues"
+  },
+  "homepage": "https://github.com/x12i/apis-hub",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@x12i/distinctive-terms": "1.1.0",
+    "@x12i/text-entities": "1.0.0"
+  }
+}