@thi.ng/text-analysis 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2025-06-09T17:24:08Z
+- **Last updated**: 2025-06-14T20:56:27Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -11,6 +11,19 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.2.0) (2025-06-14)
+
+#### 🚀 Features
+
+- add/migrate refactored tf-idf functions ([d311acc](https://github.com/thi-ng/umbrella/commit/d311acc))
+- add/update vocab & vector encoding helpers, restructure ([9e4f60c](https://github.com/thi-ng/umbrella/commit/9e4f60c))
+- add filterDocsIDF() ([f682b58](https://github.com/thi-ng/umbrella/commit/f682b58))
+- add k-mean clustering fns ([3533843](https://github.com/thi-ng/umbrella/commit/3533843))
+
+#### ♻️ Refactoring
+
+- update imports/exports ([a44be87](https://github.com/thi-ng/umbrella/commit/a44be87))
+
 ## [0.1.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/text-analysis@0.1.0) (2025-06-09)
 
 #### 🚀 Features

package/README.md

@@ -58,13 +58,16 @@ For Node.js REPL:
 const ta = await import("@thi.ng/text-analysis");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 2.44 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 3.30 KB
 
 ## Dependencies
 
 - [@thi.ng/api](https://github.com/thi-ng/umbrella/tree/develop/packages/api)
+- [@thi.ng/arrays](https://github.com/thi-ng/umbrella/tree/develop/packages/arrays)
 - [@thi.ng/bidir-index](https://github.com/thi-ng/umbrella/tree/develop/packages/bidir-index)
 - [@thi.ng/checks](https://github.com/thi-ng/umbrella/tree/develop/packages/checks)
+- [@thi.ng/distance](https://github.com/thi-ng/umbrella/tree/develop/packages/distance)
+- [@thi.ng/k-means](https://github.com/thi-ng/umbrella/tree/develop/packages/k-means)
 - [@thi.ng/strings](https://github.com/thi-ng/umbrella/tree/develop/packages/strings)
 - [@thi.ng/transducers](https://github.com/thi-ng/umbrella/tree/develop/packages/transducers)
 - [@thi.ng/vectors](https://github.com/thi-ng/umbrella/tree/develop/packages/vectors)

package/api.d.ts

@@ -0,0 +1,4 @@
+import type { BidirIndex, SerializedBidirIndex } from "@thi.ng/bidir-index";
+export type Vocab = BidirIndex<string>;
+export type SerializedVocab = SerializedBidirIndex<string>;
+//# sourceMappingURL=api.d.ts.map

package/cluster.d.ts

@@ -0,0 +1,93 @@
+import { Untransformed } from "@thi.ng/distance/untransformed";
+import { type KMeansOpts } from "@thi.ng/k-means";
+import type { ReadonlyVec } from "@thi.ng/vectors";
+import type { Vocab } from "./api.js";
+/**
+ * Jaccard distance metric wrapper for {@link kmeansDense}
+ */
+export declare const JACCARD_DIST_DENSE: Untransformed<ReadonlyVec>;
+/**
+ * k-means clustering for dense multi-hot vectors. Uses thi.ng/k-means for
+ * actual clustering and squared L2 as default distance metric.
+ * Default max. iterations = 100.
+ *
+ * @remarks
+ * Use {@link JACCARD_DIST_DENSE} for alternative distance metric.
+ *
+ * @param k
+ * @param docs
+ * @param opts
+ */
+export declare const kmeansDense: (k: number, docs: ReadonlyVec[], opts?: Partial<KMeansOpts>) => import("@thi.ng/k-means").Cluster[];
+/**
+ * k-means clustering for sparse multi-hot vectors. First converts vectors into
+ * dense versions (using {@link toDense}), then calls {@link kmeansDense} to
+ * perform the clustering.
+ *
+ * @remarks
+ * Since sparse vector sizes vary, the number of dimensions used (aka the
+ * vocabulary size) MUST be given via `opts`.
+ *
+ * @param k
+ * @param docs
+ * @param opts
+ */
+export declare const kmeansSparse: (k: number, docs: ReadonlyVec[], opts: Partial<KMeansOpts> & {
+    dim: number;
+}) => import("@thi.ng/k-means").Cluster[];
+export declare function clusterBounds(docs: ReadonlyVec[]): {
+    centroid: ReadonlyVec;
+    radius: number;
+};
+export declare function clusterBounds(docs: ReadonlyVec[], ids: number[]): {
+    centroid: ReadonlyVec;
+    radius: number;
+};
+/**
+ * Takes a vocab and array of docs encoded as dense multi-hot vectors. Computes
+ * centroid of given docs and then calls {@link centralTermsVec} to return the
+ * `k`-most central terms (or less if there're insufficient non-zero vector
+ * components).
+ *
+ * @example
+ * ```ts tangle:../export/central-terms.ts
+ * import { centralTerms, encodeAllDense } from "@thi.ng/text-analysis";
+ *
+ * const inputs = [
+ *   ["a", "b", "c"],
+ *   ["a", "b", "d", "e"],
+ *   ["b", "f", "g"],
+ *   ["a", "b", "c", "f"],
+ *   ["a", "g", "h"]
+ * ];
+ *
+ * // create vocab & encode documents into multi-hot vectors
+ * const { vocab, docs } = encodeAllDense(inputs);
+ *
+ * // extract top-4 common terms
+ * console.log(centralTerms(vocab, 4, docs));
+ * // [ "b", "a", "g", "f" ]
+ * ```
+ *
+ * @param vocab
+ * @param k
+ * @param docs
+ */
+export declare const centralTerms: (vocab: Vocab, k: number, docs: ReadonlyVec[]) => string[];
+/**
+ * Takes a vocab and dense vector representing a point in the n-dimensional
+ * space of the given vocab. Returns an array of terms corresponding to the `k`
+ * largest non-zero components of the vector (or less if there're insufficient
+ * non-zero vector components).
+ *
+ * @remarks
+ * Also see {@link centralTerms} (incl. code example).
+ *
+ * @param vocab
+ * @param k
+ * @param centroid
+ */
+export declare const centralTermsVec: (vocab: Vocab, k: number, centroid: ReadonlyVec) => string[];
+export declare const knearest: (query: ReadonlyVec, k: number, r?: number, dist?: Untransformed<ReadonlyVec>, sorted?: boolean) => import("@thi.ng/distance").KNearest<ReadonlyVec, unknown>;
+export declare const knearestDocs: (query: ReadonlyVec, k: number, docs: ReadonlyVec[], r?: number, dist?: Untransformed<ReadonlyVec>, sorted?: boolean) => [ReadonlyVec, number][];
+//# sourceMappingURL=cluster.d.ts.map

package/cluster.js

@@ -0,0 +1,51 @@
+import { argSort } from "@thi.ng/arrays/arg-sort";
+import { lookup } from "@thi.ng/arrays/lookup";
+import { knearest as $knearest } from "@thi.ng/distance/knearest";
+import { Untransformed } from "@thi.ng/distance/untransformed";
+import { kmeans } from "@thi.ng/k-means";
+import { map } from "@thi.ng/transducers/map";
+import { max } from "@thi.ng/transducers/max";
+import { transduce } from "@thi.ng/transducers/transduce";
+import { distJaccard } from "@thi.ng/vectors/dist-jaccard";
+import { distSq } from "@thi.ng/vectors/distsq";
+import { mean } from "@thi.ng/vectors/mean";
+import { toDense } from "./vec.js";
+const JACCARD_DIST_DENSE = new Untransformed(distJaccard);
+const kmeansDense = (k, docs, opts) => kmeans(k, docs, { maxIter: 100, ...opts });
+const kmeansSparse = (k, docs, opts) => kmeansDense(
+  k,
+  docs.map((x) => toDense(opts.dim, x)),
+  opts
+);
+function clusterBounds(docs, ids) {
+  if (ids) docs = lookup(docs, ids);
+  const centroid = mean([], docs);
+  return {
+    centroid,
+    radius: transduce(
+      map((x) => distSq(centroid, x)),
+      max(),
+      docs
+    )
+  };
+}
+const centralTerms = (vocab, k, docs) => centralTermsVec(vocab, k, mean([], docs));
+const centralTermsVec = (vocab, k, centroid) => vocab.getAllIDs(
+  argSort(centroid, (a, b) => b - a).slice(0, k).filter(Boolean)
+);
+const knearest = (query, k, r = Infinity, dist = JACCARD_DIST_DENSE, sorted = false) => $knearest(query, k, r, dist, sorted);
+const knearestDocs = (query, k, docs, r = Infinity, dist = JACCARD_DIST_DENSE, sorted = false) => {
+  const neighborhood = $knearest(query, k, r, dist, sorted);
+  for (let i = 0; i < docs.length; i++) neighborhood.consider(docs[i], i);
+  return neighborhood.deref().map((n) => [docs[n[1]], n[0]]);
+};
+export {
+  JACCARD_DIST_DENSE,
+  centralTerms,
+  centralTermsVec,
+  clusterBounds,
+  kmeansDense,
+  kmeansSparse,
+  knearest,
+  knearestDocs
+};

package/index.d.ts

@@ -1,10 +1,13 @@
+export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";
 export * from "./replace.js";
 export * from "./similarity.js";
 export * from "./stem.js";
 export * from "./stop-words.js";
+export * from "./tf-idf.js";
 export * from "./tokenize.js";
+export * from "./vec.js";
 export * from "./vocab.js";
 export * from "./xform.js";
 //# sourceMappingURL=index.d.ts.map

package/index.js

@@ -1,9 +1,12 @@
+export * from "./cluster.js";
 export * from "./frequencies.js";
 export * from "./ngrams.js";
 export * from "./replace.js";
 export * from "./similarity.js";
 export * from "./stem.js";
 export * from "./stop-words.js";
+export * from "./tf-idf.js";
 export * from "./tokenize.js";
+export * from "./vec.js";
 export * from "./vocab.js";
 export * from "./xform.js";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/text-analysis",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Text tokenization, transformation & analysis transducers, utilities, stop words, porter stemming, vector encodings, similarities",
   "type": "module",
   "module": "./index.js",
@@ -40,10 +40,13 @@
   },
   "dependencies": {
     "@thi.ng/api": "^8.11.29",
+    "@thi.ng/arrays": "^2.12.0",
     "@thi.ng/bidir-index": "^1.3.0",
     "@thi.ng/checks": "^3.7.9",
+    "@thi.ng/distance": "^3.0.0",
+    "@thi.ng/k-means": "^1.1.0",
     "@thi.ng/strings": "^3.9.14",
-    "@thi.ng/transducers": "^9.4.0",
+    "@thi.ng/transducers": "^9.4.1",
     "@thi.ng/vectors": "^8.3.0"
   },
   "devDependencies": {
@@ -52,17 +55,28 @@
     "typescript": "^5.8.3"
   },
   "keywords": [
+    "analysis",
+    "centroid",
+    "cluster",
     "composition",
     "decode",
+    "dense",
     "encode",
+    "frequency",
     "functional",
+    "histogram",
+    "k-means",
     "ngram",
     "pipeline",
     "similarity",
+    "sparse",
     "stem",
     "text",
+    "tf-idf",
     "tokenizer",
+    "transducer",
     "typescript",
+    "vocabulary",
     "vector"
   ],
   "publishConfig": {
@@ -83,6 +97,12 @@
     ".": {
       "default": "./index.js"
     },
+    "./api": {
+      "default": "./api.js"
+    },
+    "./cluster": {
+      "default": "./cluster.js"
+    },
     "./frequencies": {
       "default": "./frequencies.js"
     },
@@ -101,9 +121,15 @@
     "./stop-words": {
       "default": "./stop-words.js"
     },
+    "./tf-idf": {
+      "default": "./tf-idf.js"
+    },
     "./tokenize": {
       "default": "./tokenize.js"
     },
+    "./vec": {
+      "default": "./vec.js"
+    },
     "./vocab": {
       "default": "./vocab.js"
     },
@@ -115,5 +141,5 @@
     "status": "alpha",
     "year": 2021
   },
-  "gitHead": "93cdcd8db4d4669561a7f0ebc47697bdbfd04214\n"
+  "gitHead": "14e994e531d32053e948768998324d443436a542\n"
 }

package/similarity.d.ts

@@ -8,7 +8,7 @@
  * @param a
  * @param b
  */
-export declare const cosineSimilarityDense: import("@thi.ng/vectors/api").DistanceFn;
+export declare const cosineSimilarityDense: import("@thi.ng/vectors").DistanceFn;
 /**
  * Computes cosine similarity of given sparse multi-hot vectors.
  *
@@ -26,7 +26,7 @@ export declare const cosineSimilaritySparse: (a: ArrayLike<number>, b: ArrayLike
  * @param a
  * @param b
  */
-export declare const jaccardSimilarityDense: import("@thi.ng/vectors/api").DistanceFn;
+export declare const jaccardSimilarityDense: import("@thi.ng/vectors").DistanceFn;
 /**
  * Computes Jaccard similarity of given sparse multi-hot vectors.
  *
@@ -44,7 +44,7 @@ export declare const jaccardSimilaritySparse: (a: ArrayLike<number>, b: ArrayLik
  * @param a
  * @param b
  */
-export declare const dotProductDense: import("@thi.ng/vectors/api").MultiVecOpRoVV<number>;
+export declare const dotProductDense: import("@thi.ng/vectors").MultiVecOpRoVV<number>;
 /**
  * Computes dot product of the given sparse multi-hot vectors.
  *
@@ -62,7 +62,7 @@ export declare const dotProductSparse: (a: ArrayLike<number>, b: ArrayLike<numbe
  * @param a
  * @param b
  */
-export declare const distSqDense: import("@thi.ng/vectors/api").MultiVecOpRoVV<number>;
+export declare const distSqDense: import("@thi.ng/vectors").MultiVecOpRoVV<number>;
 /**
  * Computes the squared L2 distance of the given sparse multi-hot vectors.
  *

package/tf-idf.d.ts

@@ -0,0 +1,101 @@
+import type { Fn2 } from "@thi.ng/api";
+import type { Vocab } from "./api.js";
+export declare const tfCount: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+export declare const tfNormalized: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+export declare const tfLog: (vocab: Vocab, docTokens: Iterable<string>) => Map<string, number>;
+export declare const defIDF: (fnIDF: (count: number, numDocs: number) => number) => (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+export declare const idfClassic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+export declare const idfSmooth: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+export declare const idfProbabilistic: (vocab: Vocab, tokenizedDocs: string[][]) => Map<string, number>;
+/**
+ * Higher-order customizable tf-idf implementation, using provided fns for term
+ * frequency and inverse document frequency.
+ *
+ * @remarks
+ * See {@link tfidf} for default impl.
+ *
+ * Also see:
+ *
+ * - {@link tfCount}, {@link tfNormalized}, {@link tfLog}
+ * - {@link idfClassic}, {@link idfSmooth}, {@link idfProbabilistic}.
+ *
+ * References:
+ *
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+ *
+ *
+ * @param fnTF
+ * @param fnIDF
+ */
+export declare const defTFIDF: (fnTF: Fn2<Vocab, string[], Map<string, number>>, fnIDF: Fn2<Vocab, string[][], Map<string, number>>) => (vocab: Vocab, tokenizedDocs: string[][]) => {
+    doc: string[];
+    tf: Map<string, number>;
+    idf: Map<string, number>;
+    tfidf: Map<string, number>;
+}[];
+/**
+ * Default tf-idf implementation, using {@link tfNormalized} for term
+ * frequency and {@link idfClassic} for inverse document frequency
+ * calculation.
+ *
+ * @remarks
+ * References:
+ *
+ * - https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+ *
+ * Also see {@link defTFIDF}
+ *
+ * @param vocab
+ * @param tokenizedDocs
+ */
+export declare const tfidf: (vocab: Vocab, tokenizedDocs: string[][]) => {
+    doc: string[];
+    tf: Map<string, number>;
+    idf: Map<string, number>;
+    tfidf: Map<string, number>;
+}[];
+/**
+ * Takes a vocab, an array of tokenized documents and a predicate function.
+ * Computes the IDF (Inverse Document Frequency, default: {@link idfClassic})
+ * and then filters each document using supplied predicate, which is called with
+ * a single word/token and its computed IDF. Only words are kept for which the
+ * predicate succeeds.
+ *
+ * @remarks
+ * The IDF for common words is close to zero. This function can be used as a
+ * pre-processing step for improved and more efficient vocabulary construction,
+ * vector encoding (e.g. via {@link encodeDense}), clustering etc. by pre-excluding
+ * tokens which do not contribute much information.
+ *
+ * @example
+ * ```ts tangle:../export/filter-docs-idf.ts
+ * import { filterDocsIDF } from "@thi.ng/text-analysis";
+ *
+ * const docs = [
+ *   ["a", "b", "c"],
+ *   ["a", "b", "d", "e"],
+ *   ["b", "f", "g"],
+ *   ["a", "b", "c", "f"],
+ *   ["a", "g", "h"]
+ * ];
+ *
+ * // remove common words, i.e. those with an IDF below given threshold
+ * const filtered = filterDocsIDF(docs, (_, x) => x > 0.3);
+ *
+ * // show before & after
+ * for(let i = 0; i < docs.length; i++) console.log(docs[i], "=>", filtered[i]);
+ *
+ * // [ "a", "b", "c" ] => [ "c" ]
+ * // [ "a", "b", "d", "e" ] => [ "d", "e" ]
+ * // [ "b", "f", "g" ] => [ "f", "g" ]
+ * // [ "a", "b", "c", "f" ] => [ "c", "f" ]
+ * // [ "a", "g", "h" ] => [ "g", "h" ]
+ * ```
+ *
+ * @param docs
+ * @param pred
+ * @param vocab
+ * @param fnIDF
+ */
+export declare const filterDocsIDF: (docs: string[][], pred: Fn2<string, number, boolean>, vocab?: Vocab, fnIDF?: Fn2<Vocab, string[][], Map<string, number>>) => string[][];
+//# sourceMappingURL=tf-idf.d.ts.map

package/tf-idf.js

@@ -0,0 +1,63 @@
+import { transduce } from "@thi.ng/transducers/transduce";
+import { frequencies, normFrequencies } from "./frequencies.js";
+import { vocabOnly } from "./xform.js";
+import { defVocab } from "./vocab.js";
+const { log10 } = Math;
+const tfCount = (vocab, docTokens) => transduce(vocabOnly(vocab), frequencies(), docTokens);
+const tfNormalized = (vocab, docTokens) => transduce(vocabOnly(vocab), normFrequencies(), docTokens);
+const tfLog = (vocab, docTokens) => {
+  const res = transduce(vocabOnly(vocab), frequencies(), docTokens);
+  for (const [word, count] of res) res.set(word, log10(1 + count));
+  return res;
+};
+const defIDF = (fnIDF) => (vocab, tokenizedDocs) => {
+  const acc = /* @__PURE__ */ new Map();
+  for (const word of vocab.keys()) {
+    let n = 0;
+    for (const doc of tokenizedDocs) {
+      if (doc.includes(word)) n++;
+    }
+    acc.set(word, fnIDF(n, tokenizedDocs.length));
+  }
+  return acc;
+};
+const idfClassic = defIDF(
+  (count, numDocs) => Math.log10(numDocs / count)
+);
+const idfSmooth = defIDF(
+  (count, numDocs) => 1 + log10(numDocs / (1 + count))
+);
+const idfProbabilistic = defIDF(
+  (count, numDocs) => log10((numDocs - count) / count)
+);
+const defTFIDF = (fnTF, fnIDF) => (vocab, tokenizedDocs) => {
+  const idf = fnIDF(vocab, tokenizedDocs);
+  return tokenizedDocs.map((doc) => {
+    const tf = fnTF(vocab, doc);
+    const acc = /* @__PURE__ */ new Map();
+    for (const [word, f] of tf) {
+      acc.set(word, f * idf.get(word));
+    }
+    return { doc, tf, idf, tfidf: acc };
+  });
+};
+const tfidf = defTFIDF(tfNormalized, idfClassic);
+const filterDocsIDF = (docs, pred, vocab, fnIDF = idfClassic) => {
+  if (!vocab) vocab = defVocab(docs);
+  const idf = fnIDF(vocab, docs);
+  return docs.map(
+    (doc) => doc.filter((word) => vocab.has(word) && pred(word, idf.get(word)))
+  );
+};
+export {
+  defIDF,
+  defTFIDF,
+  filterDocsIDF,
+  idfClassic,
+  idfProbabilistic,
+  idfSmooth,
+  tfCount,
+  tfLog,
+  tfNormalized,
+  tfidf
+};

package/vec.d.ts

@@ -0,0 +1,150 @@
+import type { ReadonlyVec } from "@thi.ng/vectors";
+import type { Vocab } from "./api.js";
+/**
+ * Encodes the given `doc` tokens into a dense multi-hot vector using provided
+ * `vocab` (e.g. created via {@link defVocab}). The vector size is the number of
+ * items in the vocab.
+ *
+ * @remarks
+ * Also see {@link encodeSparse}.
+ *
+ * @example
+ * ```ts tangle:../export/encode-dense.ts
+ * import { defVocab, encodeDense, tokenize } from "@thi.ng/text-analysis";
+ *
+ * const vocab = defVocab(
+ *   tokenize("the quick brown fox jumps over the lazy dog")
+ * );
+ *
+ * console.log(encodeDense(vocab, tokenize("the brown dog jumps")));
+ * // [ 1, 0, 1, 0, 1, 0, 0, 1 ]
+ *
+ * console.log(encodeDense(vocab, tokenize("the lazy fox")));
+ * // [ 1, 0, 0, 1, 0, 0, 1, 0 ]
+ * ```
+ *
+ * @param vocab
+ * @param doc
+ */
+export declare const encodeDense: (vocab: Vocab, doc: Iterable<string>) => ReadonlyVec;
+/**
+ * Convenience function to create a vocabulary from given docs and encode each
+ * doc into a dense multi-hot vector (using {@link encodeDense}).
+ *
+ * @param docs
+ */
+export declare const encodeAllDense: (docs: string[][]) => {
+    vocab: Vocab;
+    docs: ReadonlyVec[];
+};
+/**
+ * Encodes the given `src` tokens into a sparse vector using provided `vocab`
+ * (created via {@link defVocab}). Only the IDs of matched tokens are stored.
+ * The returned vector size depends on the number of used/matched tokens, at
+ * most `vocab.size` (if entire vocab is used by `src`).
+ *
+ * @remarks
+ * Also see {@link encodeDense} for alternative encoding.
+ *
+ * @example
+ * ```ts tangle:../export/encode-sparse.ts
+ * import { defVocab, encodeSparse, tokenize } from "@thi.ng/text-analysis";
+ *
+ * const vocab = defVocab(
+ *   tokenize("the quick brown fox jumps over the lazy dog")
+ * );
+ *
+ * console.log(encodeSparse(vocab, tokenize("the brown dog jumps")));
+ * // [ 0, 2, 4, 7 ]
+ *
+ * console.log(encodeSparse(vocab, tokenize("the lazy fox")));
+ * // [ 0, 3, 6 ]
+ * ```
+ *
+ * @param vocab
+ * @param src
+ */
+export declare const encodeSparse: (vocab: Vocab, src: Iterable<string>) => ReadonlyVec;
+/**
+ * Convenience function to create a vocabulary from given docs and encode each
+ * doc into a sparse multi-hot vector (using {@link encodeSparse}).
+ *
+ * @param docs
+ */
+export declare const encodeAllSparse: (docs: string[][]) => {
+    vocab: Vocab;
+    docs: ReadonlyVec[];
+};
+/**
+ * Reverse op of {@link encodeDense}. Decodes dense multi-hot vector to extract
+ * tokens from provided `vocab` (created via {@link defVocab}). The returned
+ * array only contains the corresponding tokens of the vector's non-zero
+ * components.
+ *
+ * @remarks
+ * Also see {@link decodeSparse}.
+ *
+ * @example
+ * ```ts tangle:../export/decode-dense.ts
+ * import { defVocab, decodeDense, tokenize } from "@thi.ng/text-analysis";
+ *
+ * const vocab = defVocab(
+ *   tokenize("the quick brown fox jumps over the lazy dog")
+ * );
+ *
+ * console.log(decodeDense(vocab, [1, 0, 1, 0, 1, 0, 0, 1]));
+ * // [ "the", "brown", "jumps", "dog" ]
+ *
+ * console.log(decodeDense(vocab, [1, 0, 0, 1, 0, 0, 1, 0]));
+ * // [ "the", "fox", "lazy" ]
+ * ```
+ *
+ * @param vocab
+ * @param src
+ * @param sort
+ */
+export declare const decodeDense: (vocab: Vocab, vec: Iterable<number>) => string[];
+/**
+ * Reverse op of {@link encodeSparse}. Decodes sparse vector (created via
+ * {@link encodeSparse} to extract tokens from provided `vocab` (created via
+ * {@link defVocab}).
+ *
+ * @remarks
+ * Also see {@link decodeDense}.
+ *
+ * @example
+ * ```ts tangle:../export/decode-sparse.ts
+ * import { defVocab, decodeSparse, tokenize } from "@thi.ng/text-analysis";
+ *
+ * const vocab = defVocab(
+ *   tokenize("the quick brown fox jumps over the lazy dog")
+ * );
+ *
+ * console.log(decodeSparse(vocab, [0, 2, 4, 7]));
+ * // [ "the", "brown", "jumps", "dog" ]
+ *
+ * console.log(decodeSparse(vocab, [0, 3, 6]));
+ * // [ "the", "fox", "lazy" ]
+ * ```
+ *
+ * @param vocab
+ * @param src
+ * @param sort
+ */
+export declare const decodeSparse: (vocab: Vocab, vec: Iterable<number>) => string[];
+/**
+ * Converts given multi-hot sparse vector (e.g. created via {@link encodeSparse}
+ * into a dense representation.
+ *
+ * @param dim
+ * @param sparse
+ */
+export declare const toDense: (dim: number, sparse: ReadonlyVec) => ReadonlyVec;
+/**
+ * Converts given multi-hot dense vector (e.g. created via {@link encodeDense})
+ * into a sparse representation.
+ *
+ * @param dense
+ */
+export declare const toSparse: (dense: ReadonlyVec) => ReadonlyVec;
+//# sourceMappingURL=vec.d.ts.map

package/vec.js

@@ -0,0 +1,49 @@
+import { defVocab } from "./vocab.js";
+const encodeDense = (vocab, doc) => toDense(vocab.size, vocab.getAll(doc));
+const encodeAllDense = (docs) => {
+  const vocab = defVocab(docs);
+  return {
+    vocab,
+    docs: docs.map((x) => encodeDense(vocab, x))
+  };
+};
+const encodeSparse = (vocab, src) => [...vocab.getAllUnique(src)].sort((a, b) => a - b);
+const encodeAllSparse = (docs) => {
+  const vocab = defVocab(docs);
+  return {
+    vocab,
+    docs: docs.map((x) => encodeSparse(vocab, x))
+  };
+};
+const decodeDense = (vocab, vec) => {
+  const res = [];
+  let i = 0;
+  for (const x of vec) {
+    if (x) res.push(vocab.getID(i));
+    i++;
+  }
+  return res;
+};
+const decodeSparse = (vocab, vec) => vocab.getAllIDs(vec);
+const toDense = (dim, sparse) => {
+  const res = new Array(dim).fill(0);
+  for (const i of sparse) res[i] = 1;
+  return res;
+};
+const toSparse = (dense) => {
+  const res = [];
+  for (let i = 0, n = dense.length; i < n; i++) {
+    if (dense[i]) res.push(i);
+  }
+  return res;
+};
+export {
+  decodeDense,
+  decodeSparse,
+  encodeAllDense,
+  encodeAllSparse,
+  encodeDense,
+  encodeSparse,
+  toDense,
+  toSparse
+};

package/vocab.d.ts

@@ -1,10 +1,9 @@
-import { type BidirIndex, type SerializedBidirIndex } from "@thi.ng/bidir-index";
-export type Vocab = BidirIndex<string>;
-export type SerializedVocab = SerializedBidirIndex<string>;
+import type { SerializedVocab, Vocab } from "./api.js";
 /**
  * Creates a bi-directional index storing unique tokens from given `src`
  * iterable, optionally using custom `start` ID offset (default: 0). This index
- * can then be used with {@link vectorize}, {@link vectorizeSparse}.
+ * can then be used with {@link encodeDense}, {@link encodeSparse} and related
+ * functions.
  *
  * @remarks
  * This function is syntax sugar for
@@ -43,7 +42,7 @@ export type SerializedVocab = SerializedBidirIndex<string>;
  * @param src
  * @param start
  */
-export declare function defVocab(src: Iterable<string>, start?: number): Vocab;
+export declare function defVocab(src: Iterable<string> | Iterable<string>[], start?: number): Vocab;
 /**
  * (Re)creates bi-directional vocab index from previous serialized state (e.g.
  * via `vocab.toJSON()`).
@@ -51,116 +50,4 @@ export declare function defVocab(src: Iterable<string>, start?: number): Vocab;
  * @param vocab
  */
 export declare function defVocab(vocab: SerializedVocab): Vocab;
-/**
- * Encodes the given `src` tokens into a dense multi-hot vector using provided
- * `vocab` (created via {@link defVocab}). The vector size is the number of
- * items in the vocab.
- *
- * @remarks
- * Also see {@link encodeSparse}.
- *
- * @example
- * ```ts tangle:../export/encode-dense.ts
- * import { defVocab, encodeDense, tokenize } from "@thi.ng/text-analysis";
- *
- * const vocab = defVocab(
- *   tokenize("the quick brown fox jumps over the lazy dog")
- * );
- *
- * console.log(encodeDense(vocab, tokenize("the brown dog jumps")));
- * // [ 1, 0, 1, 0, 1, 0, 0, 1 ]
- *
- * console.log(encodeDense(vocab, tokenize("the lazy fox")));
- * // [ 1, 0, 0, 1, 0, 0, 1, 0 ]
- * ```
- *
- * @param vocab
- * @param src
- */
-export declare const encodeDense: (vocab: Vocab, src: Iterable<string>) => any[];
-/**
- * Encodes the given `src` tokens into a sparse vector using provided `vocab`
- * (created via {@link defVocab}). Only the IDs of matched tokens are stored.
- * The returned vector size depends on the number of used/matched tokens, at
- * most `vocab.size` (if entire vocab is used by `src`).
- *
- * @remarks
- * Also see {@link encodeDense} for alternative encoding.
- *
- * @example
- * ```ts tangle:../export/encode-sparse.ts
- * import { defVocab, encodeSparse, tokenize } from "@thi.ng/text-analysis";
- *
- * const vocab = defVocab(
- *   tokenize("the quick brown fox jumps over the lazy dog")
- * );
- *
- * console.log(encodeSparse(vocab, tokenize("the brown dog jumps")));
- * // [ 0, 2, 4, 7 ]
- *
- * console.log(encodeSparse(vocab, tokenize("the lazy fox")));
- * // [ 0, 3, 6 ]
- * ```
- *
- * @param vocab
- * @param src
- */
-export declare const encodeSparse: (vocab: Vocab, src: Iterable<string>) => number[];
-/**
- * Reverse op of {@link encodeDense}. Decodes dense multi-hot vector to extract
- * tokens from provided `vocab` (created via {@link defVocab}). The returned
- * array only contains the corresponding tokens of the vector's non-zero
- * components.
- *
- * @remarks
- * Also see {@link decodeSparse}.
- *
- * @example
- * ```ts tangle:../export/decode-dense.ts
- * import { defVocab, decodeDense, tokenize } from "@thi.ng/text-analysis";
- *
- * const vocab = defVocab(
- *   tokenize("the quick brown fox jumps over the lazy dog")
- * );
- *
- * console.log(decodeDense(vocab, [1, 0, 1, 0, 1, 0, 0, 1]));
- * // [ "the", "brown", "jumps", "dog" ]
- *
- * console.log(decodeDense(vocab, [1, 0, 0, 1, 0, 0, 1, 0]));
- * // [ "the", "fox", "lazy" ]
- * ```
- *
- * @param vocab
- * @param src
- * @param sort
- */
-export declare const decodeDense: (vocab: Vocab, vec: Iterable<number>) => string[];
-/**
- * Reverse op of {@link encodeSparse}. Decodes sparse vector (created via
- * {@link encodeSparse} to extract tokens from provided `vocab` (created via
- * {@link defVocab}).
- *
- * @remarks
- * Also see {@link decodeDense}.
- *
- * @example
- * ```ts tangle:../export/decode-sparse.ts
- * import { defVocab, decodeSparse, tokenize } from "@thi.ng/text-analysis";
- *
- * const vocab = defVocab(
- *   tokenize("the quick brown fox jumps over the lazy dog")
- * );
- *
- * console.log(decodeSparse(vocab, [0, 2, 4, 7]));
- * // [ "the", "brown", "jumps", "dog" ]
- *
- * console.log(decodeSparse(vocab, [0, 3, 6]));
- * // [ "the", "fox", "lazy" ]
- * ```
- *
- * @param vocab
- * @param src
- * @param sort
- */
-export declare const decodeSparse: (vocab: Vocab, vec: Iterable<number>) => string[];
 //# sourceMappingURL=vocab.d.ts.map

package/vocab.js

@@ -1,31 +1,13 @@
-import {
-  bidirIndexFromJSON,
-  defBidirIndex
-} from "@thi.ng/bidir-index";
+import { bidirIndexFromJSON, defBidirIndex } from "@thi.ng/bidir-index";
 import { isIterable } from "@thi.ng/checks/is-iterable";
+import { isString } from "@thi.ng/checks/is-string";
+import { mapcat } from "@thi.ng/transducers/mapcat";
 function defVocab(src, start) {
-  return isIterable(src) ? defBidirIndex(src, { start }) : bidirIndexFromJSON(src);
+  return isIterable(src) ? defBidirIndex(
+    mapcat((x) => isString(x) ? [x] : x, src),
+    { start }
+  ) : bidirIndexFromJSON(src);
 }
-const encodeDense = (vocab, src) => {
-  const vec = new Array(vocab.size).fill(0);
-  for (let i of vocab.getAll(src)) vec[i] = 1;
-  return vec;
-};
-const encodeSparse = (vocab, src) => [...vocab.getAllUnique(src)].sort((a, b) => a - b);
-const decodeDense = (vocab, vec) => {
-  const res = [];
-  let i = 0;
-  for (let x of vec) {
-    if (x) res.push(vocab.getID(i));
-    i++;
-  }
-  return res;
-};
-const decodeSparse = (vocab, vec) => vocab.getAllIDs(vec);
 export {
-  decodeDense,
-  decodeSparse,
-  defVocab,
-  encodeDense,
-  encodeSparse
+  defVocab
 };

package/xform.d.ts

@@ -1,4 +1,4 @@
-import type { Vocab } from "./vocab.js";
+import type { Vocab } from "./api.js";
 /**
  * Trnasducer to produce lowercase string.
  */