node-predict 3.0.0

package/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "node-predict",
+  "version": "3.0.0",
+  "description": "Full lightweight and fast native offline, trainable text prediction engine  for Node.js",
+  "keywords": [
+    "predict",
+    "autocomplete",
+    "text-prediction",
+    "ngram",
+    "trie",
+    "tfidf",
+    "predictjs"
+  ],
+  "license": "MIT",
+  "author": "Ismail Gidado",
+  "type": "commonjs",
+  "main": "predictjs.js",
+  "engines": {
+    "node": ">=14"
+  },
+  "scripts": {
+    "test": "node app.js"
+  }
+}

package/predictjs.js

@@ -0,0 +1,562 @@
+'use strict'
+
+/**
+ * ╔══════════════════════════════════════════════════════════════════╗
+ * ║                        PredictJS  v3.0                          ║
+ * ║   Offline text prediction engine — learns purely from text      ║
+ * ║   ──────────────────────────────────────────────────────────    ║
+ * ║   • Trie           → partial word completion                    ║
+ * ║   • Ensemble NGram → multi-size n-gram voting for accuracy      ║
+ * ║   • TF-IDF         → meaningful word scoring over filler        ║
+ * ║   • Loop guard     → prevents repetitive generation cycles      ║
+ * ║   • Predict mode   → suggest / complete / next word             ║
+ * ║   • Persistent     → save/load trained index from JSON file     ║
+ * ╚══════════════════════════════════════════════════════════════════╝
+ */
+
+const fs = require('fs')
+const path = require('path')
+
+// ─────────────────────────────────────────────
+// DEFAULT CONFIG  — tuned for best accuracy
+// ─────────────────────────────────────────────
+const DEFAULTS = {
+    // N-gram ensemble
+    nMin: 2,      // bigram — broad coverage
+    nMax: 4,      // 4-gram — specific context
+    smoothing: true,
+    smoothingAlpha: 0.05,   // light smoothing — keeps predictions tight to actual data
+
+    // TF-IDF
+    useTFIDF: true,
+    tfidfBlend: 0.5,    // 50/50 blend — balances frequency vs meaningfulness
+
+    // Ensemble weights — give more weight to larger n-grams (more specific)
+    // [weight for nMin, ..., weight for nMax]  must match nMax - nMin + 1 entries
+    ensembleWeights: [0.15, 0.35, 0.50],  // bigram=15%, trigram=35%, 4-gram=50%
+
+    // Trie
+    minWordLength: 2,
+    maxSuggestions: 5,
+
+    // Sentence completion
+    maxCompletionWords: 20,
+    completionTemp: 0.6,    // lower = more faithful to dataset patterns
+    sentenceEndTokens: ['<END>'],
+
+    // Loop / repetition guard
+    maxRepeatBigram: 2,      // stop if any 2-word sequence repeats this many times
+    penalizeRepeats: true,   // reduce score of recently used words
+
+    // Tokenizer
+    caseSensitive: false,
+    keepPunctuation: false,
+    keepNumbers: false,  // strip numbers — reduces noise
+
+    // Skip lines that look like metadata (comments, labels, etc.)
+    skipLinePatterns: [
+        /^#/,                      // comment lines
+        /^(input|response|output|user|bot|assistant|human)\s*:/i,  // dialogue labels
+        /^\s*$/,                   // blank lines
+    ],
+
+    // Dataset
+    encoding: 'utf8',
+    datasetFormat: 'auto',
+    jsonTextField: 'text',
+
+    // Index
+    indexPath: './model-index.json',
+    autoSave: false,
+}
+
+// ─────────────────────────────────────────────
+// TOKENIZER
+// ─────────────────────────────────────────────
+class Tokenizer {
+    constructor(cfg) { this.cfg = cfg }
+
+    clean(text) {
+        let t = text
+        if (!this.cfg.caseSensitive) t = t.toLowerCase()
+        if (!this.cfg.keepPunctuation) t = t.replace(/[^\w\s']/g, ' ')
+        if (!this.cfg.keepNumbers) t = t.replace(/\b\d+\b/g, ' ')
+        return t.replace(/\s+/g, ' ').trim()
+    }
+
+    tokenize(text) {
+        return this.clean(text)
+            .split(' ')
+            .map(w => w.replace(/^'+|'+$/g, ''))
+            .filter(w => w.length >= this.cfg.minWordLength)
+    }
+
+    // Split text into clean sentence token arrays
+    // Filters out any lines that look like metadata/labels
+    toSentences(text) {
+        const sentences = []
+
+        // First split by lines so we can filter whole lines
+        const lines = text.split('\n')
+
+        for (const line of lines) {
+            const trimmed = line.trim()
+
+            // Skip lines matching any skip pattern
+            const skip = this.cfg.skipLinePatterns.some(p => p.test(trimmed))
+            if (skip) continue
+
+            // Split line into sentences by punctuation, then tokenize each
+            const parts = trimmed.split(/[.!?]+/).map(s => s.trim()).filter(Boolean)
+            for (const part of parts) {
+                const tokens = this.tokenize(part)
+                if (tokens.length >= 3) sentences.push(tokens)  // minimum 3 tokens per sentence
+            }
+        }
+
+        return sentences
+    }
+
+    detectFormat(filePath, content) {
+        if (this.cfg.datasetFormat !== 'auto') return this.cfg.datasetFormat
+        const ext = path.extname(filePath).toLowerCase()
+        if (ext === '.jsonl') return 'jsonl'
+        if (ext === '.json') return 'json'
+        const t = content.trimStart()
+        if (t.startsWith('[')) return 'json'
+        if (t.startsWith('{')) return 'jsonl'
+        return 'text'
+    }
+}
+
+// ─────────────────────────────────────────────
+// TRIE
+// ─────────────────────────────────────────────
+class TrieNode {
+    constructor() { this.children = Object.create(null); this.isEnd = false; this.count = 0 }
+}
+
+class Trie {
+    constructor() { this.root = new TrieNode(); this.wordCount = 0 }
+
+    insert(word) {
+        let node = this.root
+        for (const ch of word) {
+            if (!node.children[ch]) node.children[ch] = new TrieNode()
+            node = node.children[ch]
+        }
+        if (!node.isEnd) this.wordCount++
+        node.isEnd = true
+        node.count++
+    }
+
+    // Only return words that are COMPLETE words in the trie
+    // "hi" should NOT match "higher" — user must explicitly ask for prefix
+    searchExact(word) {
+        let node = this.root
+        for (const ch of word) {
+            if (!node.children[ch]) return false
+            node = node.children[ch]
+        }
+        return node.isEnd ? node.count : false
+    }
+
+    search(prefix, limit = 5) {
+        let node = this.root
+        for (const ch of prefix) {
+            if (!node.children[ch]) return []
+            node = node.children[ch]
+        }
+        const results = []
+        this._dfs(node, prefix, results)
+        return results.sort((a, b) => b.count - a.count).slice(0, limit).map(r => r.word)
+    }
+
+    _dfs(node, cur, acc) {
+        if (node.isEnd) acc.push({ word: cur, count: node.count })
+        for (const [ch, child] of Object.entries(node.children)) this._dfs(child, cur + ch, acc)
+    }
+
+    serialize() { return { wordCount: this.wordCount, root: this._sn(this.root) } }
+    _sn(n) { return { e: n.isEnd ? 1 : 0, c: n.count, ch: Object.fromEntries(Object.entries(n.children).map(([k, v]) => [k, this._sn(v)])) } }
+
+    static deserialize(data) {
+        const t = new Trie(); t.wordCount = data.wordCount; t.root = Trie._dn(data.root); return t
+    }
+    static _dn(r) {
+        const n = new TrieNode(); n.isEnd = r.e === 1; n.count = r.c
+        for (const [k, v] of Object.entries(r.ch)) n.children[k] = Trie._dn(v)
+        return n
+    }
+}
+
+// ─────────────────────────────────────────────
+// TF-IDF
+// ─────────────────────────────────────────────
+class TFIDF {
+    constructor() { this.df = Object.create(null); this.tf = Object.create(null); this.total = 0 }
+
+    train(sentences) {
+        this.total += sentences.length
+        for (const tokens of sentences) {
+            const seen = new Set()
+            for (const t of tokens) {
+                this.tf[t] = (this.tf[t] || 0) + 1
+                if (!seen.has(t)) { this.df[t] = (this.df[t] || 0) + 1; seen.add(t) }
+            }
+        }
+    }
+
+    idf(word) { const df = this.df[word] || 0; if (!df) return 0; return Math.log((this.total + 1) / (df + 1)) + 1 }
+    score(word) { return (this.tf[word] || 0) * this.idf(word) }
+
+    serialize() { return { df: this.df, tf: this.tf, total: this.total } }
+    static deserialize(d) { const t = new TFIDF(); t.df = d.df; t.tf = d.tf; t.total = d.total; return t }
+}
+
+// ─────────────────────────────────────────────
+// ENSEMBLE N-GRAM
+// ─────────────────────────────────────────────
+class EnsembleNGram {
+    constructor(cfg) {
+        this.nMin = cfg.nMin
+        this.nMax = cfg.nMax
+        this.alpha = cfg.smoothing ? cfg.smoothingAlpha : 0
+        this.models = {}
+        for (let n = cfg.nMin; n <= cfg.nMax; n++) {
+            this.models[n] = { table: Object.create(null), vocab: new Set() }
+        }
+    }
+
+    train(sentences) {
+        for (const tokens of sentences) {
+            for (let n = this.nMin; n <= this.nMax; n++) {
+                const model = this.models[n]
+                for (const t of tokens) model.vocab.add(t)
+                for (let size = 1; size <= n; size++) {
+                    for (let i = 0; i + size < tokens.length; i++) {
+                        const ctx = tokens.slice(i, i + size).join(' ')
+                        const next = tokens[i + size]
+                        if (!model.table[ctx]) model.table[ctx] = Object.create(null)
+                        model.table[ctx][next] = (model.table[ctx][next] || 0) + 1
+                    }
+                }
+            }
+        }
+    }
+
+    predict(contextTokens, limit = 5, weights = null) {
+        const scores = Object.create(null)
+        const nSizes = Object.keys(this.models).map(Number)
+        const totalN = nSizes.reduce((a, b) => a + b, 0)
+
+        for (let idx = 0; idx < nSizes.length; idx++) {
+            const n = nSizes[idx]
+            const model = this.models[n]
+            const vocab = model.vocab.size || 1
+            const w = weights ? (weights[idx] || 1) : n / totalN
+
+            for (let size = Math.min(n, contextTokens.length); size >= 1; size--) {
+                const ctx = contextTokens.slice(-size).join(' ')
+                const dist = model.table[ctx]
+                if (!dist) continue
+                const total = Object.values(dist).reduce((a, b) => a + b, 0)
+                for (const [word, count] of Object.entries(dist)) {
+                    scores[word] = (scores[word] || 0) + ((count + this.alpha) / (total + this.alpha * vocab)) * w
+                }
+                break
+            }
+        }
+
+        return Object.entries(scores)
+            .map(([word, score]) => ({ word, score: +score.toFixed(5) }))
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit)
+    }
+
+    /**
+     * Sample with temperature + repeat penalty
+     * recentWords: Set of recently generated words to penalize
+     */
+    sample(predictions, temp = 0.6, recentWords = null) {
+        if (!predictions.length) return null
+        if (temp === 0) return predictions[0].word
+
+        let pool = predictions
+        if (recentWords && recentWords.size) {
+            // reduce score of recently used words by 60%
+            pool = predictions.map(p => ({
+                word: p.word,
+                score: recentWords.has(p.word) ? p.score * 0.4 : p.score
+            })).sort((a, b) => b.score - a.score)
+        }
+
+        const scaled = pool.map(p => ({ word: p.word, w: Math.pow(Math.max(p.score, 1e-9), 1 / temp) }))
+        const total = scaled.reduce((s, p) => s + p.w, 0)
+        let rand = Math.random() * total
+        for (const p of scaled) { rand -= p.w; if (rand <= 0) return p.word }
+        return scaled[scaled.length - 1].word
+    }
+
+    serialize() {
+        const out = { nMin: this.nMin, nMax: this.nMax, alpha: this.alpha, models: {} }
+        for (const [n, m] of Object.entries(this.models)) out.models[n] = { table: m.table, vocab: [...m.vocab] }
+        return out
+    }
+
+    static deserialize(data) {
+        const e = new EnsembleNGram({ nMin: data.nMin, nMax: data.nMax, smoothing: false, smoothingAlpha: data.alpha })
+        e.alpha = data.alpha
+        for (const [n, m] of Object.entries(data.models)) e.models[n] = { table: m.table, vocab: new Set(m.vocab) }
+        return e
+    }
+}
+
+// ─────────────────────────────────────────────
+// PIPELINE
+// ─────────────────────────────────────────────
+class Pipeline {
+    constructor(trie, ngram, tfidf, tokenizer, cfg) {
+        this.trie = trie
+        this.ngram = ngram
+        this.tfidf = tfidf
+        this.tokenizer = tokenizer
+        this.cfg = cfg
+    }
+
+    _blend(predictions) {
+        if (!this.cfg.useTFIDF || !this.tfidf) return predictions
+        if (!predictions.length) return predictions
+        const blend = this.cfg.tfidfBlend
+        const scores = predictions.map(p => this.tfidf.score(p.word))
+        const maxTF = Math.max(...scores, 1e-9)
+        return predictions.map((p, i) => ({
+            ...p,
+            score: +(p.score * (1 - blend) + (scores[i] / maxTF) * blend).toFixed(5)
+        })).sort((a, b) => b.score - a.score)
+    }
+
+    /**
+     * Suggest: if input ends with space → next words only
+     *          if input doesn't end with space → word completions for partial + next words for context
+     */
+    suggest(input, options = {}) {
+        const limit = options.limit || this.cfg.maxSuggestions
+        const endsSpace = input.endsWith(' ')
+        const tokens = this.tokenizer.tokenize(input)
+        const last = tokens[tokens.length - 1] || ''
+        const ctx = endsSpace ? tokens : tokens.slice(0, -1)
+        const result = { input, partialWord: endsSpace ? null : last, wordCompletions: [], nextWords: [] }
+
+        // Word completion — only when there is a partial word
+        if (!endsSpace && last.length >= 1) {
+            result.wordCompletions = this.trie.search(last, limit)
+        }
+
+        // Next word — always from context if we have any
+        if (ctx.length > 0) {
+            const raw = this.ngram.predict(ctx, limit * 2, this.cfg.ensembleWeights)
+            result.nextWords = this._blend(raw).slice(0, limit)
+        }
+
+        return result
+    }
+
+    /**
+     * Complete a sentence from a seed.
+     * Fixes:
+     *  - Resolves partial word at end via trie before generating
+     *  - Loop detection: tracks bigrams in generated text, stops on repeat
+     *  - Repeat penalty: reduces score of recently used words
+     */
+    complete(input, options = {}) {
+        const maxWords = options.maxWords || this.cfg.maxCompletionWords
+        const temp = options.temperature !== undefined ? options.temperature : this.cfg.completionTemp
+        const limit = options.limit || this.cfg.maxSuggestions
+
+        let tokens = this.tokenizer.tokenize(input)
+        if (!tokens.length) return { seed: input.trim(), completed: '', generated: '', wordCount: 0 }
+
+        // Resolve partial word at end if input doesn't end with space
+        if (!input.endsWith(' ') && tokens.length > 0) {
+            const completions = this.trie.search(tokens[tokens.length - 1], 1)
+            if (completions.length) tokens[tokens.length - 1] = completions[0]
+        }
+
+        const generated = []
+        const bigramCounts = Object.create(null)   // bigram loop detection
+        const recentWindow = 5                      // last N words for repeat penalty
+
+        for (let i = 0; i < maxWords; i++) {
+            const raw = this.ngram.predict(tokens, limit * 2, this.cfg.ensembleWeights)
+            const pred = this._blend(raw).slice(0, limit)
+            if (!pred.length) break
+
+            // Build recent word set for penalty
+            const recentWords = this.cfg.penalizeRepeats
+                ? new Set(generated.slice(-recentWindow))
+                : null
+
+            const next = this.ngram.sample(pred, temp, recentWords)
+            if (!next) break
+
+            // Bigram loop detection — "X Y X Y X Y" = stop
+            const bigram = tokens.length > 0 ? `${tokens[tokens.length - 1]} ${next}` : next
+            bigramCounts[bigram] = (bigramCounts[bigram] || 0) + 1
+            if (bigramCounts[bigram] >= this.cfg.maxRepeatBigram) break
+
+            tokens.push(next)
+            generated.push(next)
+        }
+
+        const seedTokens = this.tokenizer.tokenize(input.trim())
+        return {
+            seed: input.trim(),
+            completed: [...seedTokens, ...generated].join(' '),
+            generated: generated.join(' '),
+            wordCount: generated.length,
+        }
+    }
+
+    completions(input, options = {}) {
+        const count = options.count || 3
+        const seen = new Set()
+        const out = []
+        for (let i = 0; i < count * 5 && out.length < count; i++) {
+            const r = this.complete(input, options)
+            if (r.completed && !seen.has(r.completed)) { seen.add(r.completed); out.push(r) }
+        }
+        return out
+    }
+}
+
+// ─────────────────────────────────────────────
+// MAIN CLASS — PredictJS
+// ─────────────────────────────────────────────
+class PredictJS {
+    constructor(options = {}) {
+        this.config = { ...DEFAULTS, ...options }
+        this.tokenizer = new Tokenizer(this.config)
+        this.trie = new Trie()
+        this.ngram = new EnsembleNGram(this.config)
+        this.tfidf = new TFIDF()
+        this._rebuild()
+        this.trained = false
+        this.stats = { tokens: 0, sentences: 0, uniqueWords: 0, ngramContexts: 0 }
+    }
+
+    _rebuild() {
+        this.pipeline = new Pipeline(this.trie, this.ngram, this.tfidf, this.tokenizer, this.config)
+    }
+
+    // ── Training ───────────────────────────────
+
+    trainText(text) {
+        this._trainSentences(this.tokenizer.toSentences(text))
+        return this
+    }
+
+    trainArray(arr) {
+        this._trainSentences(arr.flatMap(t => this.tokenizer.toSentences(t)))
+        return this
+    }
+
+    trainFile(filePath) {
+        const absPath = path.resolve(filePath)
+        if (!fs.existsSync(absPath)) throw new Error(`File not found: ${absPath}`)
+        const content = fs.readFileSync(absPath, this.config.encoding)
+        const format = this.tokenizer.detectFormat(absPath, content)
+        let texts = []
+
+        if (format === 'text') {
+            texts = [content]
+        } else if (format === 'json') {
+            const arr = Array.isArray(JSON.parse(content)) ? JSON.parse(content) : [JSON.parse(content)]
+            texts = arr.map(i => typeof i === 'string' ? i : (i[this.config.jsonTextField] || '')).filter(Boolean)
+        } else if (format === 'jsonl') {
+            texts = content.split('\n').filter(Boolean).map(line => {
+                try { const o = JSON.parse(line); return typeof o === 'string' ? o : (o[this.config.jsonTextField] || '') } catch { return '' }
+            }).filter(Boolean)
+        }
+
+        return this.trainArray(texts)
+    }
+
+    _trainSentences(sentences) {
+        if (!sentences.length) return this
+        const allTokens = sentences.flat()
+        for (const t of allTokens) this.trie.insert(t)
+        this.ngram.train(sentences)
+        this.tfidf.train(sentences)
+
+        this.stats.tokens += allTokens.length
+        this.stats.sentences += sentences.length
+        this.stats.uniqueWords = this.trie.wordCount
+        this.stats.ngramContexts = Object.keys(this.ngram.models[this.config.nMax]?.table || {}).length
+        this.trained = true
+
+        if (this.config.autoSave) this.saveIndex()
+        return this
+    }
+
+    // ── Prediction API ─────────────────────────
+
+    suggest(input, options = {}) { this._assert(); return this.pipeline.suggest(input, options) }
+    completeWord(partial, limit) { this._assert(); return this.trie.search(partial, limit || this.config.maxSuggestions) }
+    complete(input, options = {}) { this._assert(); return this.pipeline.complete(input, options) }
+    completions(input, options = {}) { this._assert(); return this.pipeline.completions(input, options) }
+
+    nextWord(context, limit) {
+        this._assert()
+        const tokens = this.tokenizer.tokenize(context)
+        const raw = this.ngram.predict(tokens, (limit || this.config.maxSuggestions) * 2, this.config.ensembleWeights)
+        return this.pipeline._blend(raw).slice(0, limit || this.config.maxSuggestions)
+    }
+
+    // ── Index ──────────────────────────────────
+
+    saveIndex(filePath) {
+        const dest = path.resolve(filePath || this.config.indexPath)
+        const index = {
+            version: '3.0.0',
+            savedAt: new Date().toISOString(),
+            config: this.config,
+            stats: this.stats,
+            trie: this.trie.serialize(),
+            ngram: this.ngram.serialize(),
+            tfidf: this.tfidf.serialize(),
+        }
+        fs.writeFileSync(dest, JSON.stringify(index, null, 2))
+        console.log(`[PredictJS] Index saved → ${dest}`)
+        return dest
+    }
+
+    loadIndex(filePath) {
+        const src = path.resolve(filePath || this.config.indexPath)
+        if (!fs.existsSync(src)) throw new Error(`Index not found: ${src}`)
+        const index = JSON.parse(fs.readFileSync(src, 'utf8'))
+        this.config = { ...index.config, ...this.config }
+        this.stats = index.stats
+        this.trie = Trie.deserialize(index.trie)
+        this.ngram = EnsembleNGram.deserialize(index.ngram)
+        this.tfidf = TFIDF.deserialize(index.tfidf)
+        this._rebuild()
+        this.trained = true
+        console.log(`[PredictJS] Index loaded ← ${src} (${this.stats.uniqueWords} words, ${this.stats.sentences} sentences)`)
+        return this
+    }
+
+    // ── Utils ──────────────────────────────────
+
+    getStats() { return { ...this.stats, trained: this.trained, config: this.config } }
+    configure(options) { this.config = { ...this.config, ...options }; this._rebuild(); return this }
+    reset() {
+        this.trie = new Trie(); this.ngram = new EnsembleNGram(this.config)
+        this.tfidf = new TFIDF(); this._rebuild(); this.trained = false
+        this.stats = { tokens: 0, sentences: 0, uniqueWords: 0, ngramContexts: 0 }
+        return this
+    }
+    _assert() { if (!this.trained) throw new Error('[PredictJS] Model not trained. Call trainText(), trainArray(), or trainFile() first.') }
+}
+
+module.exports = { PredictJS, DEFAULTS }

package/readme.md

@@ -0,0 +1,171 @@
+# node-predict
+
+A lightweight, fully offline, trainable native text prediction engine for Node.js. Provides word completions, next-word predictions, and sentence generation using n-gram language models with TF-IDF weighting.
+
+## Quick Start
+
+Load a pre-trained model and make predictions:
+
+```js
+const { PredictJS } = require('node-predict')
+
+const predictor = new PredictJS()
+predictor.loadIndex('./model-index.json')
+
+// 1) Word completions
+console.log(predictor.completeWord('app'))
+
+// 2) Next-word predictions
+console.log(predictor.nextWord('The best way to'))
+
+// 3) Complete a sentence
+console.log(predictor.complete('Learning new skills'))
+
+// 4) Multiple completions
+console.log(predictor.completions('In my opinion', { count: 3 }))
+
+// 5) Combined suggestion (partial + next words)
+console.log(predictor.suggest('I en'))
+```
+
+---
+
+## Training Your Own Model
+
+To train the model on your custom dataset, edit `dataset.txt` with your text samples, then run `node train.js`. The training script reads `dataset.txt`, builds n-gram models, calculates TF-IDF weights, and saves the trained model to `model-index.json`.
+
+### Training Code Example
+
+Here's how `train.js` trains the model:
+
+```js
+const { PredictJS } = require('./predictjs')
+
+const DATASET_PATH = './dataset.txt'
+const INDEX_PATH = './model-index.json'
+
+const predictor = new PredictJS({
+    // N-gram range
+    nMin: 2,
+    nMax: 4,
+
+    // Smoothing — keep low for large datasets, raise towards 0.3 for small ones
+    smoothing: true,
+    smoothingAlpha: 0.05,
+
+    // Ensemble weights [bigram, trigram, 4-gram]
+    // Higher weight = more influence on predictions
+    ensembleWeights: [0.15, 0.35, 0.50],
+
+    // TF-IDF blending
+    useTFIDF: true,
+    tfidfBlend: 0.5,
+
+    // Word settings
+    minWordLength: 2,
+    maxSuggestions: 5,
+
+    // Completion settings
+    maxCompletionWords: 20,
+    completionTemp: 0.6,
+
+    // Loop prevention
+    maxRepeatBigram: 2,
+    penalizeRepeats: true,
+
+    // Strip numbers, lowercase, no punctuation
+    caseSensitive: false,
+    keepPunctuation: false,
+    keepNumbers: false,
+})
+
+console.log('Training...')
+
+try {
+    const start = Date.now()
+    predictor.trainFile(DATASET_PATH)
+    const elapsed = Date.now() - start
+    const stats = predictor.getStats()
+
+    console.log('✔ Training complete')
+    console.log('Sentences:', stats.sentences)
+    console.log('Total tokens:', stats.tokens)
+    console.log('Unique words:', stats.uniqueWords)
+    console.log('Time taken:', elapsed, 'ms')
+
+    predictor.saveIndex(INDEX_PATH)
+    console.log('✔ Model saved to', INDEX_PATH)
+
+} catch (err) {
+    console.error('✘ Training failed:', err.message)
+    process.exit(1)
+}
+```
+
+To train with your own data:
+
+1. Edit `dataset.txt` with your text samples
+2. Run `node train.js`
+3. The trained model is saved to `model-index.json`
+4. Use it as shown in the Quick Start section
+
+---
+
+## Configuration Options
+
+Edit `train.js` to customize training behavior. Common options:
+
+- **`nMin`, `nMax`** — n-gram range (default `2–4`). Controls context window size for predictions.
+- **`smoothingAlpha`** — smoothing strength for unseen word pairs (default `0.05`). Keeps low for large datasets, raise towards 0.3 for small ones.
+- **`ensembleWeights`** — weights for [bigram, trigram, 4-gram] predictions (default `[0.15, 0.35, 0.50]`). Higher weights give more influence.
+- **`tfidfBlend`** — blend between raw frequency and TF-IDF scoring (default `0.5`). Higher values make results less common-word-heavy.
+- **`maxCompletionWords`** — maximum words to generate (default `20`).
+- **`completionTemp`** — sampling temperature for completions (default `0.6`).
+- **`minWordLength`** — minimum word length to consider (default `2`).
+- **`maxSuggestions`** — maximum suggestions to return (default `5`).
+
+After editing options, re-run `node train.js` to retrain with new settings.
+
+---
+
+## API Reference
+
+### `predictor.completeWord(prefix)`
+Returns word completions for a given prefix.
+
+### `predictor.nextWord(context)`
+Predicts the next word given preceding context.
+
+### `predictor.complete(seed)`
+Generates a complete sentence starting from a seed phrase.
+
+### `predictor.completions(context, options)`
+Returns multiple completion options with customizable count.
+
+### `predictor.suggest(partial)`
+Combined suggestion: completes partial word + predicts next words.
+
+---
+
+## Temperature (Sampling)
+
+The `completionTemp` option controls creativity of generated suggestions:
+
+- **`0.0–0.3`** — very predictable, deterministic
+- **`0.6`** — balanced (default), good for most uses
+- **`0.8–1.0`** — more creative and varied, may diverge from dataset style
+
+---
+
+## Tips & Troubleshooting
+
+- **Poor predictions?** Add more in-style text to `dataset.txt` and retrain.
+- **Too many common words?** Increase dataset variety or raise `tfidfBlend` towards 1.0.
+- **No matches found?** Ensure your seed text matches your dataset language/style.
+- **Slow training?** Large datasets may take longer; consider using representative samples.
+- **Memory usage?** Larger n-gram ranges and bigger datasets consume more memory.
+
+---
+
+Author: Ismail Gidado  
+License: MIT