ohlcv-indicators 3.5.4 → 3.5.42

package/src/machine-learning/ml-utilities.js

@@ -0,0 +1,148 @@
+export const oneHotEncode = (idx, size) => {
+    const vec = new Uint8Array(size)
+    vec[idx] = 1
+    return vec
+}
+
+export const logMlTraining = ({featureCols, flatFeaturesColLen, type, trainingSize}) => {
+
+    const oneHotTxt = (featureCols.length !== flatFeaturesColLen) ? ' (including flat one-hot encoded array values) ' : ''
+    console.log(`---\nInitialized "${type}" with ${flatFeaturesColLen} features ${oneHotTxt} and ${trainingSize} rows: \n${JSON.stringify(featureCols)}\n\n`)
+}
+
+export const normalizeMinMax = (value, min, max, [validMin, validMax]) => {
+    const clamped = Math.min(Math.max(value, min), max)
+    return ((clamped - min) / (max - min)) * (validMax - validMin) + validMin
+}
+  
+export const normalizeZScore = (value, mean, std) => {
+    return std === 0 ? 0 : (value - mean) / std
+}
+
+export const findGroupsFunc = (findGroups, scaledGroups) => {
+
+    const output = []
+
+    if(findGroups.length > 0)
+    {
+        for(let g = 0; g < findGroups.length; g++)
+        {
+            const group = findGroups[g]
+            const {type, size = null, groupName = null} = group
+
+            let groupKey = (size !== null) ? `${type}_${size}` : `${type}_${groupName}`
+
+            if(!scaledGroups.hasOwnProperty(groupKey)) throw new Error(`Scaled group (${groupKey}) not found for regressor.options.findGroups[${g}]: ${JSON.stringify(group)}`)
+            output.push(...scaledGroups[groupKey])
+        }
+    }
+
+    return output
+}
+
+export const computeFlatFeaturesLen = (featureCols, verticalOhlcv, index) => {
+  
+  let flatFeaturesColLen = 0
+
+  for (const key of featureCols) {
+
+    if(!verticalOhlcv.hasOwnProperty(key)) throw new Error(`Property "${key}" not found in "verticalOhlcv".`)
+
+    const val = verticalOhlcv[key][index]
+
+    if(typeof val === 'undefined' || val === null || Number.isNaN(val)) throw new Error(`Invalid value of property "${key}".`)
+
+    if (key.startsWith('one_hot_') || key.startsWith('pca_')) {
+
+      const colSize = verticalOhlcv[key][index].length
+
+      flatFeaturesColLen += colSize
+
+    } else {
+
+      flatFeaturesColLen += 1
+
+    }
+  }
+
+  return flatFeaturesColLen
+}
+
+
+export const countUniqueLabels = Y => {
+  // If Y[0] isn't an array, treat Y as a single-column 2D array
+  const data = Array.isArray(Y[0]) ? Y : Y.map(v => [v]);
+
+  // Number of columns in our (possibly wrapped) data
+  const nCols = data[0].length;
+
+  // Sum up the unique values in each column
+  const totalLabels = Array.from({ length: nCols }).reduce((sum, _, col) => {
+    const uniques = new Set(data.map(row => row[col]));
+    return sum + uniques.size;
+  }, 0);
+
+  // Return count + 1 (same as before)
+  return totalLabels + 1;
+}
+
+
+export const getFeaturedKeys = ({trainingCols, findGroups, verticalOhlcv, scaledGroups}) => {
+
+  const featureCols = [...trainingCols, ...(findGroupsFunc(findGroups, scaledGroups))]
+
+
+  if (featureCols.length === 0) {
+    return []
+  }
+
+  let keyNotFound = false
+
+  // sanity‐check that all features exist
+  for(let x = 0; x < featureCols.length; x++)
+  {
+    const featureKey = featureCols[x]
+
+    if(!verticalOhlcv.hasOwnProperty(featureKey)) {
+      keyNotFound = true
+      break
+    }
+  }
+  
+  if(keyNotFound) return []
+
+  return featureCols
+}
+
+export const updateClassifierMetrics = ({ metrics, trueLabel, predictedLabel }) => {
+  // Increment overall counters
+  metrics.total = (metrics.total || 0) + 1
+  metrics.correct = (metrics.correct || 0)
+
+  // Initialize per‐label stats container if needed
+  metrics.labels = metrics.labels || {}
+
+  const labelKey = predictedLabel.toString()
+  // Ensure we have a slot for this label
+  if (!metrics.labels[labelKey]) {
+    metrics.labels[labelKey] = { total: 0, correct: 0, accuracy: 0 }
+  }
+
+  // Update this label’s total predictions
+  metrics.labels[labelKey].total++
+
+  // If correct, bump both overall and per-label correct counters
+  if (predictedLabel === trueLabel) {
+    metrics.correct++
+    metrics.labels[labelKey].correct++
+  }
+
+  // Recompute overall accuracy
+  metrics.accuracy = metrics.correct / metrics.total
+
+  // Recompute accuracy for each label
+  for (const key in metrics.labels) {
+    const stats = metrics.labels[key]
+    stats.accuracy = stats.correct / stats.total
+  }
+}

package/src/machine-learning/pca.js

@@ -1,103 +1,96 @@
-import { PCA } from "ml-pca";
-import { validateBoolean, validateArrayOptions, validateNumber } from "../utilities/validators.js";
+import { getFeaturedKeys, computeFlatFeaturesLen, findGroupsFunc } from "./ml-utilities.js";
+import { buildTrainX } from "./trainX.js";
+import { areKeyValuesValid } from "../core-functions/pushToMain.js";
 
-export const pca = main => {
+const algo = 'pca'
 
-    const {inputParams, invalidValueIndex, len, verticalOhlcv} = main
-    const startIndex = invalidValueIndex + 1
+export const pca = (main, index, {prefix, trainingSize, findGroups, trainingCols, lookbackAbs, modelArgs, filterCallback}) => {
 
-    for(const rowParams of inputParams)
-    {
-        const {key: indicatorName, params} = rowParams
-
-        if(indicatorName !== 'scaler') continue
-
-        const [size, keyNames, options] = params
-
-        const {type, group, lag, pca: pcaOptions} = options
-
-        const {
-            showSource = false,
-            storeModel = false,
-            isCovarianceMatrix = false,
-            method = 'NIPALS', //SVD, covarianceMatrix or NIPALS
-            center = true,
-            scale = false,
-            nCompNIPALS = 2,
-            ignoreZeroVariance = false
-        } = pcaOptions
-
-        if(!group || pcaOptions === null) continue
-
-        validateBoolean(showSource, 'showSource', 'scaler.options.pca.showSource')
-        validateBoolean(storeModel, 'storeModel', 'scaler.options.pca.storeModel')
-        validateBoolean(isCovarianceMatrix, 'isCovarianceMatrix', 'scaler.options.pca.isCovarianceMatrix')
-        validateBoolean(center, 'center', 'scaler.options.pca.center')
-        validateBoolean(scale, 'scale', 'scaler.options.pca.scale')
-        validateBoolean(ignoreZeroVariance, 'ignoreZeroVariance', 'scaler.options.pca.ignoreZeroVariance')
-        validateArrayOptions(['SVD', 'NIPALS', 'covarianceMatrix'], method, 'scaler.options.pca.method')
-        validateNumber(nCompNIPALS, {min: 1}, 'nCompNIPALS', 'scaler.options.pca.nCompNIPALS')
-
-        const scaledKeyNames = keyNames.map(v => `${type}_${size}_${v}`)
-        const laggedKeys = []
-
-        for (const k of scaledKeyNames) {
-            for (let step = 1; step <= lag; step++) {
-                laggedKeys.push(`${k}_lag_${step}`)
-            }
-        }
+    const {verticalOhlcv, scaledGroups, len, instances, ML, invalidValueIndex} = main
 
-        scaledKeyNames.push(...laggedKeys)
+    const startIndex = (invalidValueIndex + 2)
+    const allModels = ML.models
 
-        const diffLen = len - startIndex
-        const numCols = scaledKeyNames.length
-        const tempArr = new Array(diffLen)
-
-        for (let i = startIndex; i < len; i++) {
+    if(index === startIndex)
+    {
+        if(!instances.hasOwnProperty(algo)) instances[algo] = {}
 
-            const row = new Float64Array(numCols);
+        instances[algo][prefix] = {
+            flatFeaturesColLen: 0,
+            featureCols: [],
+            X: []
+        }
+        
+        verticalOhlcv[prefix] = new Array(len).fill(null)
+        main.notNumberKeys.add(prefix)
+    }
+    else if(index < startIndex || index < lookbackAbs)
+    {
+        return
+    }
+    else if(filterCallback(verticalOhlcv, index) === false)
+    {
+        return
+    }
+    else if(index + 1 === len) 
+    {
 
-            let j = 0
+        if(instances[algo][prefix].featureCols.length === 0)
+        {
+            const inputFeatures = [...trainingCols, ...(findGroupsFunc(findGroups, scaledGroups))]
 
-            for (const key of scaledKeyNames) {
-                row[j++] = verticalOhlcv[key][i]
-            }
+            throw new Error(`Some of the provided features in ${prefix} features where not found in "verticalOhlcv": ${JSON.stringify(inputFeatures)}`)
+        }
 
-            tempArr[i - startIndex] = row
+        //last execution
+        for(const featureKey of instances[algo][prefix].featureCols)
+        {
+            if(!verticalOhlcv.hasOwnProperty(featureKey)) throw new Error(`Feature "${featureKey}" not found in verticalOhlcv for pca.`)
         }
 
-        if(!showSource)
+        if(instances[algo][prefix].X.length < trainingSize)
         {
-            for (const key of scaledKeyNames) {
-                delete verticalOhlcv[key]
-            }
+            const requiredDatapoints = instances[algo][prefix].X.length - trainingSize
+            throw new Error(`The current "trainingSize" at "${prefix}" requires at least ${requiredDatapoints} more datapoints. Try adding more input ohlcv rows or reducing the "trainingSize" by ${requiredDatapoints}.`)
         }
+    } 
 
-        const verticalKey = `pca_${type}_${size}`
+    const dataSetInstance = instances[algo][prefix]
 
-        const pca = new PCA(tempArr, {
-            isCovarianceMatrix,
-            method,
-            center,
-            scale,
-            nCompNIPALS,
-            ignoreZeroVariance
-        })
+    if(dataSetInstance.flatFeaturesColLen === 0)
+    {
+        dataSetInstance.featureCols = getFeaturedKeys({trainingCols, findGroups, verticalOhlcv, scaledGroups})
 
-        if(storeModel)
+        if(areKeyValuesValid(main, index, dataSetInstance.featureCols))
         {
-            main.models[verticalKey] = pca.toJSON()
+            dataSetInstance.flatFeaturesColLen = computeFlatFeaturesLen(dataSetInstance.featureCols, verticalOhlcv, index)
+        } else {
+            return
         }
-        
-        const predictions = pca.predict(tempArr).to2DArray()
+    }
 
-        verticalOhlcv[verticalKey] = new Array(len).fill(null)
+    if(dataSetInstance.flatFeaturesColLen === 0) return
+    
+    const {X: xRows, featureCols, flatFeaturesColLen} = dataSetInstance
 
-        for(let x = 0; x < predictions.length; x++)
-        {
-            verticalOhlcv[verticalKey][x + startIndex] = predictions[x]
-        }
+    const trainX = buildTrainX({featureCols, flatFeaturesColLen, type: algo, index, lookbackAbs, main})
+
+    if (trainX == null) return
 
+    if(allModels.hasOwnProperty(prefix))
+    {
+
+        const futureRow = allModels[prefix].predict([trainX]).to2DArray()
+
+        verticalOhlcv[prefix][index] = futureRow[0]
     }
 
+    xRows.push(trainX)
+
+    if (xRows.length > trainingSize) xRows.shift()
+
+    if(xRows.length === trainingSize)
+    {
+        allModels[prefix] = new ML.classes.PCA(xRows, modelArgs) 
+    }
 }

package/src/machine-learning/regressor.js

@@ -0,0 +1,227 @@
+import { getFeaturedKeys, computeFlatFeaturesLen, logMlTraining, findGroupsFunc } from './ml-utilities.js'
+import { buildTrainX } from './trainX.js'
+import { areKeyValuesValid } from '../core-functions/pushToMain.js'
+
+const algo = 'regressor'
+
+export const regressor = (main, index, trainingSize, {target, predictions, retrain, trainingCols, findGroups, type, modelArgs, precompute, filterCallback, modelConfig, modelClass}) => {
+
+    const {lookbackAbs, prefix, flatX, flatY} = precompute
+    const {verticalOhlcv, len, instances, scaledGroups, invalidValueIndex, ML} = main
+    const allModels = ML.models
+    const startIndex = (invalidValueIndex + 1)
+
+    if(index === startIndex)
+    {
+        if (!instances.hasOwnProperty(algo)) instances.regressor = {}
+        if(!instances.regressor.hasOwnProperty(prefix)) instances.regressor[prefix] = {}
+
+        const expectedLoops = (flatY) ? predictions : 1 //
+
+        instances.regressor[prefix] = {
+            expectedLoops,
+            trainingSize,
+            isTrained: new Array(expectedLoops).fill(false),
+            shouldRetrain: retrain,
+            featureCols: [],
+            flatFeaturesColLen: 0,
+            X: [],
+            Y: Array.from({ length: expectedLoops }, () => []),
+        }
+
+        for(let x = 0; x < predictions; x++)
+        {
+            const predictionKey = `${prefix}_${(x+1)}`
+            verticalOhlcv[predictionKey] = new Float64Array(len).fill(NaN)
+
+            if(!allModels.hasOwnProperty(predictionKey))
+            {
+                allModels[predictionKey] = []
+            }
+        }
+    } 
+    else if(index < startIndex || index < lookbackAbs)
+    {
+        return
+    }
+    else if(filterCallback(verticalOhlcv, index) === false)
+    {
+        return
+    }
+    else if(index + 1 === len) 
+    {
+        if(instances.regressor[prefix].featureCols.length === 0)
+        {
+            const inputFeatures = [...trainingCols, ...(findGroupsFunc(findGroups, scaledGroups))]
+
+            throw new Error(`Some of the provided  features in ${prefix} features where not found in "verticalOhlcv": ${JSON.stringify(inputFeatures)}`)
+        }
+
+        //last execution
+        for(const featureKey of instances.regressor[prefix].featureCols)
+        {
+            if(!verticalOhlcv.hasOwnProperty(featureKey)) throw new Error(`Feature "${featureKey}" not found in verticalOhlcv for ${prefix}.`)
+        }
+
+        if(instances.regressor[prefix].X.length < trainingSize)
+        {
+            const requiredDatapoints = instances.regressor[prefix].X.length - trainingSize
+            throw new Error(`The current "trainingSize" at "${prefix}" requires at least ${requiredDatapoints} more datapoints. Try adding more input ohlcv rows or reducing the "trainingSize" by ${requiredDatapoints}.`)
+        }
+    }
+
+    const dataSetInstance = instances.regressor[prefix]
+
+    if(dataSetInstance.flatFeaturesColLen === 0)
+    {
+        dataSetInstance.featureCols = getFeaturedKeys({trainingCols, findGroups, verticalOhlcv, scaledGroups})
+
+        if(areKeyValuesValid(main, index, dataSetInstance.featureCols))
+        {
+            dataSetInstance.flatFeaturesColLen = computeFlatFeaturesLen(dataSetInstance.featureCols, verticalOhlcv, index)
+
+            logMlTraining({
+                featureCols: dataSetInstance.featureCols, 
+                flatFeaturesColLen: dataSetInstance.flatFeaturesColLen, 
+                type, 
+                trainingSize
+            })
+
+        } else {
+            return
+        }
+    }
+    
+    if(dataSetInstance.flatFeaturesColLen === 0) return
+
+    if(!verticalOhlcv.hasOwnProperty(target))
+    {
+        throw new Error(`Target property "${target}" not found in verticalOhlcv for regressor.`)
+    }
+
+    const {
+        expectedLoops,
+        X: xRows, 
+        Y, 
+        flatFeaturesColLen, 
+        featureCols, 
+        shouldRetrain
+    } = dataSetInstance
+
+    if(!featureCols.includes(target))
+    {
+        throw new Error(`Target property "${target}" not found in ${type} features: ${JSON.stringify(featureCols)}`)
+    }
+    
+    const trainX = (flatX) ? verticalOhlcv[target][index] : buildTrainX({featureCols, flatFeaturesColLen, type, index, lookbackAbs, main})
+
+    if (flatX) {
+        if (!Number.isFinite(trainX)) return;   // or `trainX == null`
+    } else {
+        if (trainX == null) return;
+    }
+
+    let trainY = []
+
+    for (let i = 0; i < predictions; i++) {
+        const v = verticalOhlcv[target][index + i + 1]
+
+        if (!Number.isFinite(v)) {
+            break
+        }
+
+        trainY.push(v)
+    }
+
+    const {train, unsupervised} = modelConfig
+
+    //if univariable Y (flatY) a model is created for each prediction
+    for(let loopIdx = 0; loopIdx < expectedLoops; loopIdx++)
+    {
+        const predictionKey = `${prefix}_${(loopIdx+1)}`
+        const yRows = Y[loopIdx]
+        const currTrainY =  (flatY) ? (typeof trainY[loopIdx] === 'undefined') ? null : trainY[loopIdx] : trainY
+        const isTrained = dataSetInstance.isTrained[loopIdx]
+
+        const shouldPredict = allModels.hasOwnProperty(predictionKey) && isTrained && allModels[predictionKey].length === expectedLoops
+
+        //predicts using previously saved models even if current currTrainY is not available
+        if(shouldPredict)
+        {
+            const currModel = allModels[predictionKey][0]
+            const futureRow = (unsupervised) ? currModel.predict(predictions)[0] : currModel.predict([trainX])[0]
+            
+            if(flatY)
+            {
+                if(Number.isNaN(futureRow)) throw new Error(`Prediction of ${type} at index ${index} was expecting a number.`)
+
+                main.pushToMain({index, key: predictionKey, value: futureRow})
+            }
+            else {
+
+                if (!Array.isArray(futureRow) || futureRow.length !== predictions) {
+                    throw new Error(
+                        `Prediction output of "${type}" at index ${index} was expecting an array of values.`
+                    )
+                }
+
+                for(let preIdx = 0; preIdx < predictions; preIdx++) {
+                    main.pushToMain({index, key: `${prefix}_${(preIdx+1)}`, value: futureRow[preIdx]})
+                }
+            }
+        }
+
+        if((index + predictions + 1) > len) continue
+        if(trainY.length < predictions) continue
+        if(currTrainY === null) continue
+
+        if(flatY)
+        {
+            if (typeof currTrainY !== 'number' || Number.isNaN(currTrainY))
+            {
+                throw new Error(`currTrainY must return number, got ${typeof currTrainY} at index ${index}`)
+            }
+        } else {
+            if (!Array.isArray(currTrainY)) {
+                throw new Error(`currTrainY must return an array, got ${typeof currTrainY} at index ${index}`)
+            }
+
+            if ((currTrainY.length !== predictions)) {
+
+                throw new Error(`currTrainY length (${currTrainY.length}) ≠ "options.predictions" (${predictions}) for regressor "${type}"`)
+            }           
+        }
+
+        // enqueue
+        yRows.push(currTrainY)
+        if (yRows.length > trainingSize) yRows.shift()
+
+        if(loopIdx === 0)
+        {
+            xRows.push(trainX)
+            if (xRows.length > trainingSize) xRows.shift()
+        }
+
+        const shouldTrainModel = shouldRetrain || !isTrained
+
+        if (shouldTrainModel && xRows.length === trainingSize && yRows.length === trainingSize) {
+
+            const trainedModel = train({modelClass, xRows, yRows, modelArgs, uniqueLabels: 0})
+
+            if(unsupervised)
+            {
+                allModels[predictionKey] = [trainedModel]
+            } else {
+                allModels[predictionKey].push(trainedModel)
+
+                if(allModels[predictionKey].length > predictions)
+                {
+                    allModels[predictionKey].shift()
+                }
+            }
+
+            dataSetInstance.isTrained[loopIdx] = true
+        }
+
+    }
+}