@thi.ng/text-analysis 0.2.0 → 0.3.1

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-06-14T20:56:27Z
+- **Last updated**: 2025-06-15T18:41:17Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,19 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+### [0.3.1](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.3.1) (2025-06-15)
+
+#### 🩹 Bug fixes
+
+- update pkg exports ([ea72b9f](https://github.com/thi-ng/umbrella/commit/ea72b9f))
+
+## [0.3.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.3.0) (2025-06-15)
+
+#### 🚀 Features
+
+- update kmeansDense ([d35b6bd](https://github.com/thi-ng/umbrella/commit/d35b6bd))
+  - update results to include original `docs` for each cluster
+
 ## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.2.0) (2025-06-14)
 
 #### 🚀 Features

package/README.md

@@ -19,6 +19,7 @@
 - [Installation](#installation)
 - [Dependencies](#dependencies)
 - [API](#api)
+  - [Code example](#code-example)
 - [Authors](#authors)
 - [License](#license)
 
@@ -58,7 +59,7 @@ For Node.js REPL:
 const ta = await import("@thi.ng/text-analysis");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 3.30 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 3.33 KB
 
 ## Dependencies
 
@@ -78,7 +79,89 @@ Note: @thi.ng/api is in _most_ cases a type-only import (not used at runtime)
 
 [Generated API docs](https://docs.thi.ng/umbrella/text-analysis/)
 
-TODO
+### Code example
+
+```ts tangle:export/readme-1.ts
+import { files, readJSON } from "@thi.ng/file-io";
+import {
+    centralTerms,
+    encodeAllDense,
+    filterDocsIDF,
+    JACCARD_DIST_DENSE,
+    kmeansDense,
+    sortedFrequencies,
+} from "@thi.ng/text-analysis";
+
+// read package files of all ~210 umbrella libraries
+const packages = [...files("packages", "package.json")].map((file) => {
+    const { name, keywords = [] } = readJSON(file);
+    return { id: name, tags: keywords };
+});
+
+// remove tags from each package which are too common and don't contribute
+// meaningful information (using inverse document frequency)
+const filteredTags = filterDocsIDF(
+    packages.map((x) => x.tags),
+    // filter predicate using arbitrary threshold
+    (_, idf) => idf > 1
+);
+
+// create an index of all remaining unique tags (vocab) and use this index to
+// encode each package's tags as dense multi-hot vectors
+const { vocab: allTags, docs: encodedPkgs } = encodeAllDense(filteredTags);
+
+// show index/vocab size. all document vectors have this size/dimensionality
+console.log("unique tags", allTags.size);
+// unique tags 747
+
+// show the top 10 tags used across all packages
+console.log("top 10 tags:", centralTerms(allTags, 10, encodedPkgs));
+// top 10 tags: [
+//   "iterator", "canvas", "typedarray", "hiccup", "tree",
+//   "graph", "parser", "codegen", "random", "vector"
+// ]
+
+// alternative approach (using a reducer) to extract top 10 tags with counts
+console.log(
+    "sorted freq:",
+    sortedFrequencies(filteredTags.flat()).slice(0, 10)
+);
+// sorted freq: [
+//   ["iterator", 20], ["canvas", 20], ["typedarray", 19], ["tree", 18], ["hiccup", 18],
+//   ["graph", 17], ["parser", 16], ["codegen", 16], ["vector", 15], ["random", 15]
+// ]
+
+// cluster packages using k-means with Jaccard distance metric
+const clusters = kmeansDense(20, encodedPkgs, { dist: JACCARD_DIST_DENSE });
+
+// display cluster info
+for (let { id, docs, items } of clusters) {
+    console.log(`cluster #${id} size: ${docs.length}`);
+    console.log(`top 5 tags:`, centralTerms(allTags, 5, docs));
+    console.log(`pkgs:`, items.map((i) => packages[i].id).join(", "));
+}
+
+// cluster #0 size: 10
+// top 5 tags: [ "color", "image", "rgb", "palette", "css" ]
+// pkgs: @thi.ng/blurhash, @thi.ng/color, @thi.ng/color-palettes, @thi.ng/hdiff, @thi.ng/imago,
+// @thi.ng/meta-css, @thi.ng/pixel, @thi.ng/pixel-analysis, @thi.ng/pixel-dominant-colors
+// @thi.ng/porter-duff
+//
+// cluster #1 size: 10
+// top 5 tags: [ "vector", "simulation", "time", "physics", "interpolation" ]
+// pkgs: @thi.ng/boids, @thi.ng/cellular, @thi.ng/dlogic, @thi.ng/dual-algebra,
+// @thi.ng/pixel-flow, @thi.ng/text-analysis, @thi.ng/timestep, @thi.ng/vclock,
+// @thi.ng/vectors, @thi.ng/wasm-api-schedule
+//
+// cluster #2 size: 19
+// top 5 tags: [ "canvas", "shader", "webgl", "shader-ast", "codegen" ]
+// pkgs: @thi.ng/canvas, @thi.ng/dl-asset, @thi.ng/hdom-canvas, @thi.ng/hiccup-css,
+// @thi.ng/hiccup-html-parse, @thi.ng/imgui, @thi.ng/layout, @thi.ng/mime,
+// @thi.ng/rdom-canvas, @thi.ng/scenegraph, @thi.ng/shader-ast, @thi.ng/shader-ast-glsl,
+// @thi.ng/shader-ast-js, @thi.ng/shader-ast-optimize, @thi.ng/wasm-api-canvas,
+// @thi.ng/wasm-api-webgl, @thi.ng/webgl, @thi.ng/webgl-msdf, @thi.ng/webgl-shadertoy
+// ...
+```
 
 ## Authors
 

package/cluster.d.ts

@@ -18,7 +18,12 @@ export declare const JACCARD_DIST_DENSE: Untransformed<ReadonlyVec>;
  * @param docs
  * @param opts
  */
-export declare const kmeansDense: (k: number, docs: ReadonlyVec[], opts?: Partial<KMeansOpts>) => import("@thi.ng/k-means").Cluster[];
+export declare const kmeansDense: (k: number, docs: ReadonlyVec[], opts?: Partial<KMeansOpts>) => {
+    docs: ReadonlyVec[];
+    id: number;
+    centroid: ReadonlyVec;
+    items: number[];
+}[];
 /**
  * k-means clustering for sparse multi-hot vectors. First converts vectors into
  * dense versions (using {@link toDense}), then calls {@link kmeansDense} to
@@ -34,7 +39,12 @@ export declare const kmeansDense: (k: number, docs: ReadonlyVec[], opts?: Partia
  */
 export declare const kmeansSparse: (k: number, docs: ReadonlyVec[], opts: Partial<KMeansOpts> & {
     dim: number;
-}) => import("@thi.ng/k-means").Cluster[];
+}) => {
+    docs: ReadonlyVec[];
+    id: number;
+    centroid: ReadonlyVec;
+    items: number[];
+}[];
 export declare function clusterBounds(docs: ReadonlyVec[]): {
     centroid: ReadonlyVec;
     radius: number;

package/cluster.js

@@ -11,7 +11,10 @@ import { distSq } from "@thi.ng/vectors/distsq";
 import { mean } from "@thi.ng/vectors/mean";
 import { toDense } from "./vec.js";
 const JACCARD_DIST_DENSE = new Untransformed(distJaccard);
-const kmeansDense = (k, docs, opts) => kmeans(k, docs, { maxIter: 100, ...opts });
+const kmeansDense = (k, docs, opts) => kmeans(k, docs, { maxIter: 100, ...opts }).map((cluster) => ({
+  ...cluster,
+  docs: lookup(docs, cluster.items)
+}));
 const kmeansSparse = (k, docs, opts) => kmeansDense(
   k,
   docs.map((x) => toDense(opts.dim, x)),

package/index.d.ts

@@ -1,3 +1,4 @@
+export * from "./api.js";
 export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";

package/index.js

@@ -1,3 +1,4 @@
+export * from "./api.js";
 export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/text-analysis",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities",
   "type": "module",
   "module": "./index.js",
@@ -141,5 +141,5 @@
     "status": "alpha",
     "year": 2021
   },
-  "gitHead": "14e994e531d32053e948768998324d443436a542\n"
+  "gitHead": "2e3adc4a5b737e21da697d6e935150c0855844dc\n"
 }

package/tf-idf.d.ts

@@ -1,11 +1,56 @@
 import type { Fn2 } from "@thi.ng/api";
 import type { Vocab } from "./api.js";
+/**
+ * TF weighting function for {@link defTFIDF}. Computes {@link frequencies} for
+ * given words/tokens (only includes those defined in `vocab`).
+ */
 export declare const tfCount: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+/**
+ * TF weighting function for {@link defTFIDF}. Computes {@link normFrequencies}
+ * for given words/tokens (only includes those defined in `vocab`).
+ */
 export declare const tfNormalized: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+/**
+ * TF weighting function for {@link defTFIDF}. First computes
+ * {@link frequencies} for given words/tokens (only includes those defined in
+ * `vocab`), then transforms each value via `log10(1 + count)`.
+ */
 export declare const tfLog: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
-export declare const defIDF: (fnIDF: (count: number, numDocs: number) => number) => (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * Higher order Inverse Document Frequency, using provided weighting strategy
+ * function.
+ *
+ * @remarks
+ * Also see {@link defTFIDF} for full tf-idf implementation.
+ *
+ * References:
+ *
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Inverse_document_frequency
+ *
+ * Provided IDF impls for use with this function:
+ *
+ * - {@link idfClassic}
+ * - {@link idfSmooth}
+ * - {@link idfProbabilistic}
+ *
+ * @param fnIDF
+ */
+export declare const defIDF: (fnIDF: (docsWithTerm: number, numDocs: number) => number) => (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `log10(numDocs / docsWithTerm)`
+ */
 export declare const idfClassic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `1 + log10(numDocs / (1 + docsWithTerm))`
+ */
 export declare const idfSmooth: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * IDF weighting function for {@link defIDF} and {@link defTFIDF}. Computes:
+ * `log10((numDocs - docsWithTerm) / docsWithTerm)`
+ */
 export declare const idfProbabilistic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
 /**
  * Higher-order customizable tf-idf implementation, using provided fns for term
@@ -43,7 +88,7 @@ export declare const defTFIDF: (fnTF: Fn2<Vocab, string[], Map<string, number>>,
  *
  * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
  *
- * Also see {@link defTFIDF}
+ * Also see {@link defTFIDF}, {@link defIDF}.
  *
  * @param vocab
  * @param tokenizedDocs

package/tf-idf.js

@@ -13,22 +13,22 @@ const tfLog = (vocab, docTokens) => {
 const defIDF = (fnIDF) => (vocab, tokenizedDocs) => {
   const acc = /* @__PURE__ */ new Map();
   for (const word of vocab.keys()) {
-    let n = 0;
+    let count = 0;
     for (const doc of tokenizedDocs) {
-      if (doc.includes(word)) n++;
+      if (doc.includes(word)) count++;
     }
-    acc.set(word, fnIDF(n, tokenizedDocs.length));
+    acc.set(word, fnIDF(count, tokenizedDocs.length));
   }
   return acc;
 };
 const idfClassic = defIDF(
-  (count, numDocs) => Math.log10(numDocs / count)
+  (docsWithTerm, numDocs) => log10(numDocs / docsWithTerm)
 );
 const idfSmooth = defIDF(
-  (count, numDocs) => 1 + log10(numDocs / (1 + count))
+  (docsWithTerm, numDocs) => 1 + log10(numDocs / (1 + docsWithTerm))
 );
 const idfProbabilistic = defIDF(
-  (count, numDocs) => log10((numDocs - count) / count)
+  (docsWithTerm, numDocs) => log10((numDocs - docsWithTerm) / docsWithTerm)
 );
 const defTFIDF = (fnTF, fnIDF) => (vocab, tokenizedDocs) => {
   const idf = fnIDF(vocab, tokenizedDocs);