@thi.ng/text-analysis 0.3.0 → 0.4.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-06-15T12:37:24Z
+- **Last updated**: 2025-06-18T12:01:21Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,23 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.4.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.4.0) (2025-06-18)
+
+#### 🚀 Features
+
+- add `filterDocsFrequency()` ([6ac1f90](https://github.com/thi-ng/umbrella/commit/6ac1f90))
+
+#### ⏱ Performance improvements
+
+- minor update kmeansDense() ([ebd5618](https://github.com/thi-ng/umbrella/commit/ebd5618))
+  - internal use `lookupUnsafe()`
+
+### [0.3.1](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.3.1) (2025-06-15)
+
+#### 🩹 Bug fixes
+
+- update pkg exports ([ea72b9f](https://github.com/thi-ng/umbrella/commit/ea72b9f))
+
 ## [0.3.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.3.0) (2025-06-15)
 
 #### 🚀 Features

package/README.md

@@ -19,6 +19,7 @@
 - [Installation](#installation)
 - [Dependencies](#dependencies)
 - [API](#api)
+  - [Code example](#code-example)
 - [Authors](#authors)
 - [License](#license)
 
@@ -58,7 +59,7 @@ For Node.js REPL:
 const ta = await import("@thi.ng/text-analysis");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 3.30 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 3.37 KB
 
 ## Dependencies
 
@@ -78,7 +79,89 @@ Note: @thi.ng/api is in _most_ cases a type-only import (not used at runtime)
 
 [Generated API docs](https://docs.thi.ng/umbrella/text-analysis/)
 
-TODO
+### Code example
+
+```ts tangle:export/readme-1.ts
+import { files, readJSON } from "@thi.ng/file-io";
+import {
+    centralTerms,
+    encodeAllDense,
+    filterDocsIDF,
+    JACCARD_DIST_DENSE,
+    kmeansDense,
+    sortedFrequencies,
+} from "@thi.ng/text-analysis";
+
+// read package files of all ~210 umbrella libraries
+const packages = [...files("packages", "package.json")].map((file) => {
+    const { name, keywords = [] } = readJSON(file);
+    return { id: name, tags: keywords };
+});
+
+// remove tags from each package which are too common and don't contribute
+// meaningful information (using inverse document frequency)
+const filteredTags = filterDocsIDF(
+    packages.map((x) => x.tags),
+    // filter predicate using arbitrary threshold
+    (_, idf) => idf > 1
+);
+
+// create an index of all remaining unique tags (vocab) and use this index to
+// encode each package's tags as dense multi-hot vectors
+const { vocab: allTags, docs: encodedPkgs } = encodeAllDense(filteredTags);
+
+// show index/vocab size. all document vectors have this size/dimensionality
+console.log("unique tags", allTags.size);
+// unique tags 747
+
+// show the top 10 tags used across all packages
+console.log("top 10 tags:", centralTerms(allTags, 10, encodedPkgs));
+// top 10 tags: [
+//   "iterator", "canvas", "typedarray", "hiccup", "tree",
+//   "graph", "parser", "codegen", "random", "vector"
+// ]
+
+// alternative approach (using a reducer) to extract top 10 tags with counts
+console.log(
+    "sorted freq:",
+    sortedFrequencies(filteredTags.flat()).slice(0, 10)
+);
+// sorted freq: [
+//   ["iterator", 20], ["canvas", 20], ["typedarray", 19], ["tree", 18], ["hiccup", 18],
+//   ["graph", 17], ["parser", 16], ["codegen", 16], ["vector", 15], ["random", 15]
+// ]
+
+// cluster packages using k-means with Jaccard distance metric
+const clusters = kmeansDense(20, encodedPkgs, { dist: JACCARD_DIST_DENSE });
+
+// display cluster info
+for (let { id, docs, items } of clusters) {
+    console.log(`cluster #${id} size: ${docs.length}`);
+    console.log(`top 5 tags:`, centralTerms(allTags, 5, docs));
+    console.log(`pkgs:`, items.map((i) => packages[i].id).join(", "));
+}
+
+// cluster #0 size: 10
+// top 5 tags: [ "color", "image", "rgb", "palette", "css" ]
+// pkgs: @thi.ng/blurhash, @thi.ng/color, @thi.ng/color-palettes, @thi.ng/hdiff, @thi.ng/imago,
+// @thi.ng/meta-css, @thi.ng/pixel, @thi.ng/pixel-analysis, @thi.ng/pixel-dominant-colors
+// @thi.ng/porter-duff
+//
+// cluster #1 size: 10
+// top 5 tags: [ "vector", "simulation", "time", "physics", "interpolation" ]
+// pkgs: @thi.ng/boids, @thi.ng/cellular, @thi.ng/dlogic, @thi.ng/dual-algebra,
+// @thi.ng/pixel-flow, @thi.ng/text-analysis, @thi.ng/timestep, @thi.ng/vclock,
+// @thi.ng/vectors, @thi.ng/wasm-api-schedule
+//
+// cluster #2 size: 19
+// top 5 tags: [ "canvas", "shader", "webgl", "shader-ast", "codegen" ]
+// pkgs: @thi.ng/canvas, @thi.ng/dl-asset, @thi.ng/hdom-canvas, @thi.ng/hiccup-css,
+// @thi.ng/hiccup-html-parse, @thi.ng/imgui, @thi.ng/layout, @thi.ng/mime,
+// @thi.ng/rdom-canvas, @thi.ng/scenegraph, @thi.ng/shader-ast, @thi.ng/shader-ast-glsl,
+// @thi.ng/shader-ast-js, @thi.ng/shader-ast-optimize, @thi.ng/wasm-api-canvas,
+// @thi.ng/wasm-api-webgl, @thi.ng/webgl, @thi.ng/webgl-msdf, @thi.ng/webgl-shadertoy
+// ...
+```
 
 ## Authors
 

package/frequencies.d.ts

@@ -1,3 +1,4 @@
+import type { Fn, Fn2 } from "@thi.ng/api";
 import { frequencies as $freq } from "@thi.ng/transducers/frequencies";
 import { normFrequenciesAuto as $norm } from "@thi.ng/transducers/norm-frequencies-auto";
 import { sortedFrequencies as $sorted } from "@thi.ng/transducers/sorted-frequencies";
@@ -59,4 +60,43 @@ export declare const normFrequencies: typeof $norm;
  * ```
  */
 export declare const sortedFrequencies: typeof $sorted;
+/**
+ * Takes an array of tokenized documents, a histogram function (`frequencies`)
+ * and a predicate function (`pred`). First computes the combined histogram of
+ * terms/works in all given docs using `frequencies`, then filters each document
+ * using supplied predicate, which is called with a single word/token and its
+ * computed frequency. Only words are kept for which the predicate succeeds.
+ *
+ * @remarks
+ * See {@link frequencies} and {@link normFrequencies} for histogram creation.
+ *
+ * @example
+ * ```ts tangle:../export/filter-docs-frequency.ts
+ * import { filterDocsFrequency, frequencies } from "@thi.ng/text-analysis";
+ *
+ * const docs = [
+ *   ["a", "b", "c"],
+ *   ["a", "b", "d", "e"],
+ *   ["b", "f", "g"],
+ *   ["a", "b", "c", "f"],
+ *   ["a", "g", "h"]
+ * ];
+ *
+ * // only keep words which occur more than once
+ * const filtered = filterDocsFrequency(docs, frequencies, (_, x) => x > 1);
+ *
+ * // show before & after
+ * for(let i = 0; i < docs.length; i++) console.log(docs[i], "=>", filtered[i]);
+ * // [ "a", "b", "c" ] => [ "a", "b", "c" ]
+ * // [ "a", "b", "d", "e" ] => [ "a", "b" ]
+ * // [ "b", "f", "g" ] => [ "b", "f", "g" ]
+ * // [ "a", "b", "c", "f" ] => [ "a", "b", "c", "f" ]
+ * // [ "a", "g", "h" ] => [ "a", "g" ]
+ * ```
+ *
+ * @param docs
+ * @param frequencies
+ * @param pred
+ */
+export declare const filterDocsFrequency: (docs: string[][], frequencies: Fn<Iterable<string>, Map<string, number>>, pred: Fn2<string, number, boolean>) => string[][];
 //# sourceMappingURL=frequencies.d.ts.map

package/frequencies.js

@@ -4,7 +4,17 @@ import { sortedFrequencies as $sorted } from "@thi.ng/transducers/sorted-frequen
 const frequencies = $freq;
 const normFrequencies = $norm;
 const sortedFrequencies = $sorted;
+const filterDocsFrequency = (docs, frequencies2, pred) => {
+  const histogram = frequencies2(docs.flat());
+  return docs.map(
+    (doc) => doc.filter((word) => {
+      const freq = histogram.get(word);
+      return freq !== void 0 && pred(word, freq);
+    })
+  );
+};
 export {
+  filterDocsFrequency,
   frequencies,
   normFrequencies,
   sortedFrequencies

package/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./api.js";
 export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";

package/index.js

@@ -1,3 +1,4 @@
+export * from "./api.js";
 export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/text-analysis",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities",
   "type": "module",
   "module": "./index.js",
@@ -40,14 +40,14 @@
   },
   "dependencies": {
     "@thi.ng/api": "^8.11.29",
-    "@thi.ng/arrays": "^2.12.0",
+    "@thi.ng/arrays": "^2.13.0",
     "@thi.ng/bidir-index": "^1.3.0",
     "@thi.ng/checks": "^3.7.9",
-    "@thi.ng/distance": "^3.0.0",
-    "@thi.ng/k-means": "^1.1.0",
-    "@thi.ng/strings": "^3.9.14",
-    "@thi.ng/transducers": "^9.4.1",
-    "@thi.ng/vectors": "^8.3.0"
+    "@thi.ng/distance": "^3.0.1",
+    "@thi.ng/k-means": "^1.1.1",
+    "@thi.ng/strings": "^3.9.15",
+    "@thi.ng/transducers": "^9.4.2",
+    "@thi.ng/vectors": "^8.3.1"
   },
   "devDependencies": {
     "esbuild": "^0.25.5",
@@ -141,5 +141,5 @@
     "status": "alpha",
     "year": 2021
   },
-  "gitHead": "4635a24acc2623894887ca31189fdffda87ff9d3\n"
+  "gitHead": "b076434a497b291ad33e81b1a15f6a71e2c82cc2\n"
 }

package/tf-idf.d.ts

@@ -1,11 +1,56 @@
 import type { Fn2 } from "@thi.ng/api";
 import type { Vocab } from "./api.js";
+/**
+ * TF weighting function for {@link defTFIDF}. Computes {@link frequencies} for
+ * given words/tokens (only includes those defined in `vocab`).
+ */
 export declare const tfCount: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+/**
+ * TF weighting function for {@link defTFIDF}. Computes {@link normFrequencies}
+ * for given words/tokens (only includes those defined in `vocab`).
+ */
 export declare const tfNormalized: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+/**
+ * TF weighting function for {@link defTFIDF}. First computes
+ * {@link frequencies} for given words/tokens (only includes those defined in
+ * `vocab`), then transforms each value via `log10(1 + count)`.
+ */
 export declare const tfLog: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
-export declare const defIDF: (fnIDF: (count: number, numDocs: number) => number) => (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * Higher order Inverse Document Frequency, using provided weighting strategy
+ * function.
+ *
+ * @remarks
+ * Also see {@link defTFIDF} for full tf-idf implementation.
+ *
+ * References:
+ *
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Inverse_document_frequency
+ *
+ * Provided IDF impls for use with this function:
+ *
+ * - {@link idfClassic}
+ * - {@link idfSmooth}
+ * - {@link idfProbabilistic}
+ *
+ * @param fnIDF
+ */
+export declare const defIDF: (fnIDF: (docsWithTerm: number, numDocs: number) => number) => (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `log10(numDocs / docsWithTerm)`
+ */
 export declare const idfClassic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `1 + log10(numDocs / (1 + docsWithTerm))`
+ */
 export declare const idfSmooth: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `log10((numDocs - docsWithTerm) / docsWithTerm)`
+ */
 export declare const idfProbabilistic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
 /**
  * Higher-order customizable tf-idf implementation, using provided fns for term
@@ -43,7 +88,7 @@ export declare const defTFIDF: (fnTF: Fn2<Vocab, string[], Map<string, number>>,
  *
  * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
  *
- * Also see {@link defTFIDF}
+ * Also see {@link defTFIDF}, {@link defIDF}.
  *
  * @param vocab
  * @param tokenizedDocs

package/tf-idf.js

@@ -13,22 +13,22 @@ const tfLog = (vocab, docTokens) => {
 const defIDF = (fnIDF) => (vocab, tokenizedDocs) => {
   const acc = /* @__PURE__ */ new Map();
   for (const word of vocab.keys()) {
-    let n = 0;
+    let count = 0;
     for (const doc of tokenizedDocs) {
-      if (doc.includes(word)) n++;
+      if (doc.includes(word)) count++;
     }
-    acc.set(word, fnIDF(n, tokenizedDocs.length));
+    acc.set(word, fnIDF(count, tokenizedDocs.length));
   }
   return acc;
 };
 const idfClassic = defIDF(
-  (count, numDocs) => Math.log10(numDocs / count)
+  (docsWithTerm, numDocs) => log10(numDocs / docsWithTerm)
 );
 const idfSmooth = defIDF(
-  (count, numDocs) => 1 + log10(numDocs / (1 + count))
+  (docsWithTerm, numDocs) => 1 + log10(numDocs / (1 + docsWithTerm))
 );
 const idfProbabilistic = defIDF(
-  (count, numDocs) => log10((numDocs - count) / count)
+  (docsWithTerm, numDocs) => log10((numDocs - docsWithTerm) / docsWithTerm)
 );
 const defTFIDF = (fnTF, fnIDF) => (vocab, tokenizedDocs) => {
   const idf = fnIDF(vocab, tokenizedDocs);