@thi.ng/text-analysis 0.3.1 → 0.4.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-06-15T18:41:17Z
+- **Last updated**: 2025-06-18T12:01:21Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,17 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.4.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.4.0) (2025-06-18)
+
+#### 🚀 Features
+
+- add `filterDocsFrequency()` ([6ac1f90](https://github.com/thi-ng/umbrella/commit/6ac1f90))
+
+#### ⏱ Performance improvements
+
+- minor update kmeansDense() ([ebd5618](https://github.com/thi-ng/umbrella/commit/ebd5618))
+  - internal use `lookupUnsafe()`
+
 ### [0.3.1](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.3.1) (2025-06-15)
 
 #### 🩹 Bug fixes

package/README.md

@@ -59,7 +59,7 @@ For Node.js REPL:
 const ta = await import("@thi.ng/text-analysis");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 3.33 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 3.37 KB
 
 ## Dependencies
 

package/frequencies.d.ts

@@ -1,3 +1,4 @@
+import type { Fn, Fn2 } from "@thi.ng/api";
 import { frequencies as $freq } from "@thi.ng/transducers/frequencies";
 import { normFrequenciesAuto as $norm } from "@thi.ng/transducers/norm-frequencies-auto";
 import { sortedFrequencies as $sorted } from "@thi.ng/transducers/sorted-frequencies";
@@ -59,4 +60,43 @@ export declare const normFrequencies: typeof $norm;
  * ```
  */
 export declare const sortedFrequencies: typeof $sorted;
+/**
+ * Takes an array of tokenized documents, a histogram function (`frequencies`)
+ * and a predicate function (`pred`). First computes the combined histogram of
+ * terms/works in all given docs using `frequencies`, then filters each document
+ * using supplied predicate, which is called with a single word/token and its
+ * computed frequency. Only words are kept for which the predicate succeeds.
+ *
+ * @remarks
+ * See {@link frequencies} and {@link normFrequencies} for histogram creation.
+ *
+ * @example
+ * ```ts tangle:../export/filter-docs-frequency.ts
+ * import { filterDocsFrequency, frequencies } from "@thi.ng/text-analysis";
+ *
+ * const docs = [
+ *   ["a", "b", "c"],
+ *   ["a", "b", "d", "e"],
+ *   ["b", "f", "g"],
+ *   ["a", "b", "c", "f"],
+ *   ["a", "g", "h"]
+ * ];
+ *
+ * // only keep words which occur more than once
+ * const filtered = filterDocsFrequency(docs, frequencies, (_, x) => x > 1);
+ *
+ * // show before & after
+ * for(let i = 0; i < docs.length; i++) console.log(docs[i], "=>", filtered[i]);
+ * // [ "a", "b", "c" ] => [ "a", "b", "c" ]
+ * // [ "a", "b", "d", "e" ] => [ "a", "b" ]
+ * // [ "b", "f", "g" ] => [ "b", "f", "g" ]
+ * // [ "a", "b", "c", "f" ] => [ "a", "b", "c", "f" ]
+ * // [ "a", "g", "h" ] => [ "a", "g" ]
+ * ```
+ *
+ * @param docs
+ * @param frequencies
+ * @param pred
+ */
+export declare const filterDocsFrequency: (docs: string[][], frequencies: Fn<Iterable<string>, Map<string, number>>, pred: Fn2<string, number, boolean>) => string[][];
 //# sourceMappingURL=frequencies.d.ts.map

package/frequencies.js

@@ -4,7 +4,17 @@ import { sortedFrequencies as $sorted } from "@thi.ng/transducers/sorted-frequen
 const frequencies = $freq;
 const normFrequencies = $norm;
 const sortedFrequencies = $sorted;
+const filterDocsFrequency = (docs, frequencies2, pred) => {
+  const histogram = frequencies2(docs.flat());
+  return docs.map(
+    (doc) => doc.filter((word) => {
+      const freq = histogram.get(word);
+      return freq !== void 0 && pred(word, freq);
+    })
+  );
+};
 export {
+  filterDocsFrequency,
   frequencies,
   normFrequencies,
   sortedFrequencies

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/text-analysis",
-  "version": "0.3.1",
+  "version": "0.4.0",
   "description": "Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities",
   "type": "module",
   "module": "./index.js",
@@ -40,14 +40,14 @@
   },
   "dependencies": {
     "@thi.ng/api": "^8.11.29",
-    "@thi.ng/arrays": "^2.12.0",
+    "@thi.ng/arrays": "^2.13.0",
     "@thi.ng/bidir-index": "^1.3.0",
     "@thi.ng/checks": "^3.7.9",
-    "@thi.ng/distance": "^3.0.0",
-    "@thi.ng/k-means": "^1.1.0",
-    "@thi.ng/strings": "^3.9.14",
-    "@thi.ng/transducers": "^9.4.1",
-    "@thi.ng/vectors": "^8.3.0"
+    "@thi.ng/distance": "^3.0.1",
+    "@thi.ng/k-means": "^1.1.1",
+    "@thi.ng/strings": "^3.9.15",
+    "@thi.ng/transducers": "^9.4.2",
+    "@thi.ng/vectors": "^8.3.1"
   },
   "devDependencies": {
     "esbuild": "^0.25.5",
@@ -141,5 +141,5 @@
     "status": "alpha",
     "year": 2021
   },
-  "gitHead": "2e3adc4a5b737e21da697d6e935150c0855844dc\n"
+  "gitHead": "b076434a497b291ad33e81b1a15f6a71e2c82cc2\n"
 }