ohlcv-indicators 3.5.41 → 3.5.42

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ohlcv-indicators",
-  "version": "3.5.41",
+  "version": "3.5.42",
   "description": "A JavaScript library featuring optimized trading indicators for OHLCV arrays [{open, high, low, close, value}]. This library is optimized for backtesting and trading bot development.",
   "main": "index.js",
   "type": "module",
@@ -12,7 +12,6 @@
   "author": "https://jaimelias.com/",
   "license": "ISC",
   "dependencies": {
-    "ml-pca": "^4.1.1",
     "trading-signals": "^5.0.4",
     "xy-scale": "^1.4.1"
   },
@@ -20,8 +19,18 @@
     "@babel/core": "^7.25.2",
     "@babel/plugin-proposal-class-properties": "^7.18.6",
     "@babel/preset-env": "^7.25.3",
+    "arima": "^0.2.5",
     "babel-loader": "^9.1.3",
     "dotenv": "^16.4.5",
+    "merge-multi-timeframe": "^1.1.6",
+    "ml-cart": "^2.1.1",
+    "ml-fnn": "^5.0.0",
+    "ml-knn": "^3.0.0",
+    "ml-naivebayes": "^4.0.0",
+    "ml-pca": "^4.1.1",
+    "ml-random-forest": "^2.1.0",
+    "ml-regression-multivariate-linear": "^2.0.4",
+    "ml-regression-simple-linear": "^3.0.1",
     "terser-webpack-plugin": "^5.3.14",
     "webpack": "^5.93.0",
     "webpack-cli": "^5.1.4"

package/src/core-functions/mainLoop.js

@@ -1,6 +1,9 @@
+import { areKeyValuesValid } from "./pushToMain.js";
 import { validateInputParams } from "../utilities/validators.js";
 import { rsi } from "../oscillators/rsi.js";
+import { stochastic } from "../oscillators/stochastic.js";
 import { movingAverages } from "../moving-averages/movingAverages.js";
+import { vidya } from "../moving-averages/vidya.js";
 import { macd } from "../moving-averages/macd.js";
 import { relativeVolume } from "../moving-averages/relativeVolume.js";
 import { donchianChannels } from "../moving-averages/donchianChannel.js";
@@ -11,31 +14,48 @@ import { crossPairs } from "../studies/findCrosses.js";
 import { dateTime } from "../studies/dateTime.js";
 import { scaler } from "../machine-learning/scaler.js";
 import { atr } from "../volatility/atr.js";
+import { mapCols } from "../studies/mapCols.js";
 
 import { buildArray } from "../utilities/assignTypes.js";
 import {  numberFormater } from "../utilities/numberUtilities.js";
 import { dateFormaters } from "../utilities/dateUtilities.js";
 
 import { pca } from "../machine-learning/pca.js";
+import { regressor } from "../machine-learning/regressor.js";
+import { classifier } from "../machine-learning/classifier.js";
 
 // Map indicator keys to their respective functions
-const indicatorFunctions = {
+const mainFunctions = {
   dateTime,
   rsi,
+  stochastic,
   atr,
   ema: movingAverages,
   sma: movingAverages,
+  vidya,
   macd,
   relativeVolume,
   donchianChannels,
   bollingerBands,
   volumeOscillator,
-  scaler
+  mapCols,
+  scaler,
+  crossPairs,
+  lag
 };
 
-
 export const mainLoop = (input, main) => {
-  const { len, inputParams, priceBased, precisionMultiplier, arrayTypes, verticalOhlcv, verticalOhlcvKeyNames, inputTypes, chunkProcess, scaledGroups } = main;
+  const { len, 
+    inputParams, 
+    precisionMultiplier, 
+    arrayTypes, 
+    verticalOhlcv, 
+    verticalOhlcvKeyNames, 
+    inputTypes, 
+    chunkProcess, 
+    notNumberKeys, 
+    processSecondaryLoop,
+  } = main;
 
   validateInputParams(inputParams, len)
 
@@ -43,7 +63,6 @@ export const mainLoop = (input, main) => {
   {
     verticalOhlcv[key] = buildArray(arrayTypes[key], len)
   }
-
   // Process each row in the input
   for (let chunkStart = 0; chunkStart < len; chunkStart += chunkProcess) {
 
@@ -60,7 +79,7 @@ export const mainLoop = (input, main) => {
           {
             let formatedValue = numberFormater[formaterKey](value)
 
-            if(precisionMultiplier > 1 && priceBased.has(key))
+            if(precisionMultiplier > 1 && !notNumberKeys.has(key))
             {
               formatedValue = formatedValue * precisionMultiplier
             }
@@ -76,22 +95,57 @@ export const mainLoop = (input, main) => {
             main.pushToMain({index, key, value})
           }
         }
-      
-        // Run all indicator functions except for the ones processed later
-        for (const { key, params } of inputParams) {
-          if (key === 'lag' || key === 'crossPairs') continue;
-          // resolve fn on-demand, no per-item object allocation here
-          indicatorFunctions[key](main, index, ...params)
-        }
     
         // Process these indicators separately (ensuring their execution order)
-        lag(main, index)
-        crossPairs(main, index)
+        for (const { key, params, order } of inputParams)
+        {
+          for(let orderIdx = 0; orderIdx <= 9; orderIdx++)
+          {
+            if(typeof order !== 'number') throw new Error(`order property of ${key} not found: ${params}`)
+            if(orderIdx !== order) continue
+            mainFunctions[key](main, index, ...params)
+          }
+        }
+      
         input[index] = null //flusing data
+
+        const keyNames = Object.keys(verticalOhlcv)
+
+        // if any value at this index is missing/NaN/null, mark it invalid
+        if (!areKeyValuesValid(main, index, keyNames)) {
+          main.invalidValueIndex = index
+        }
       }
   }
 
-  pca(main)
+  const isSecondaryLoopNeeded = inputParams.some(({ key }) => key === 'regressor' || key === 'classifier' || key === 'pca')
+
+  if(isSecondaryLoopNeeded || processSecondaryLoop)
+  {
+    secondaryLoop(main)
+  }
 
   verticalOhlcvKeyNames.push(...Object.keys(verticalOhlcv))
+  
+}
+
+export const secondaryLoop = (main) => {
+  const {inputParams, len, invalidValueIndex} = main
+
+  const startIndex = invalidValueIndex + 1
+
+  for(let index = startIndex; index < len; index++)
+  {
+      for (const { key, params, order } of inputParams)
+      {
+        for(let orderIdx = 10; orderIdx <= 20; orderIdx++) {
+          if(typeof order !== 'number') throw new Error(`order property of ${key} not found: ${params}`)
+          if(orderIdx !== order) continue
+          if(key === 'pca') pca(main, index, ...params)
+          if(key === 'classifier') classifier(main, index, ...params)
+          if(key === 'regressor') regressor(main, index, ...params)
+        }
+      }
+  }
+
 }

package/src/core-functions/pushToMain.js

@@ -2,14 +2,35 @@ export const pushToMain = ({main, index, key, value}) => {
 
     const {verticalOhlcv} = main
 
-    if(value === null || typeof value === 'undefined' || Number.isNaN(value))
-    {
-        main.invalidValueIndex = index
-        verticalOhlcv[key][index] = value
-        return false
-    }
-
     verticalOhlcv[key][index] = value    
 
     return true
+}
+
+export const areKeyValuesValid = (main, index, keyNames) => {
+    
+    if(keyNames.length === 0) return false
+
+    const {verticalOhlcv} = main
+
+    let output = true
+
+    for(let x = 0 ; x < keyNames.length; x++)
+    {
+        const key = keyNames[x]
+        if(!verticalOhlcv.hasOwnProperty(key)) {
+            output = false
+            break
+        }
+
+        const val = verticalOhlcv[key][index]
+
+        if(typeof val === 'undefined' || val === null || Number.isNaN(val))
+        {
+            output = false
+            break
+        }
+    }
+
+    return output
 }

package/src/machine-learning/classifier.js

@@ -0,0 +1,267 @@
+import { getFeaturedKeys, computeFlatFeaturesLen, countUniqueLabels, logMlTraining, updateClassifierMetrics, findGroupsFunc } from "./ml-utilities.js"
+import { buildTrainX } from "./trainX.js"
+import { areKeyValuesValid } from "../core-functions/pushToMain.js"
+
+const algo = 'classifier'
+
+export const classifier = (
+  main,
+  index,
+  trainingSize,
+  {
+    yCallback,
+    trainingCols,
+    findGroups = [],
+    retrain,
+    type,
+    modelArgs,
+    horizon,
+    precompute,
+    predictions,
+    filterCallback,
+    modelConfig,
+    modelClass
+  }
+) => {
+  const { lookbackAbs, prefix, flatY } = precompute
+  const { verticalOhlcv, len, instances, scaledGroups, invalidValueIndex, ML } = main
+  const allModels = ML.models
+  const startIndex = (invalidValueIndex + 1)
+  
+  // ─── INITIALIZATION ───────────────────────────────────────────────
+  if(index === startIndex) {
+
+    // prepare instance storage
+    if (!instances.hasOwnProperty(algo)) instances[algo] = {}
+    if(!instances[algo].hasOwnProperty(prefix)) instances[algo][prefix] = {}
+
+    const expectedLoops = (flatY) ? predictions : 1 //
+
+    instances[algo][prefix] = {
+
+      expectedLoops,
+      isTrained: new Array(expectedLoops).fill(false),
+      uniqueLabels: new Array(expectedLoops).fill(0),
+      shouldRetrain: retrain,
+      featureCols: [],
+      flatFeaturesColLen: 0,
+      X: [],
+      Y: Array.from({ length: expectedLoops }, () => []),
+    }
+
+    for (let i = 0; i < predictions; i++) {
+      const predictionKey = `${prefix}_${i + 1}`
+
+      verticalOhlcv[predictionKey] = (type === 'KNN') ? new Array(len).fill(null) : new Float32Array(len).fill(NaN)
+      ML.metrics[predictionKey] = {accuracy: {}, total: 0, correct: 0, labels: {}}
+
+      if(!allModels.hasOwnProperty(predictionKey))
+      {
+        allModels[predictionKey] = []
+      }
+    }
+  } 
+  else if(index < startIndex || index < lookbackAbs)
+  {
+    return
+  }
+  else if(filterCallback(verticalOhlcv, index) === false)
+  {
+      return
+  }
+  else if(index + 1 === len) 
+  {
+
+      if(instances[algo][prefix].featureCols.length === 0)
+      {
+        const inputFeatures = [...trainingCols, ...(findGroupsFunc(findGroups, scaledGroups))]
+
+        throw new Error(`Some of the provided features in ${algo} "${prefix}" features where not found in "verticalOhlcv": ${JSON.stringify(inputFeatures)}`)
+      }
+
+      //last execution
+      for(const featureKey of instances[algo][prefix].featureCols)
+      {
+          if(!verticalOhlcv.hasOwnProperty(featureKey)) throw new Error(`Feature "${featureKey}" not found in verticalOhlcv for ${prefix}.`)
+      }
+
+      if(instances[algo][prefix].X.length < trainingSize)
+      {
+        const requiredDatapoints = instances[algo][prefix].X.length - trainingSize
+        throw new Error(`The current "trainingSize" at ${algo} "${prefix}" requires at least ${requiredDatapoints} more datapoints. Try adding more input ohlcv rows or reducing the "trainingSize" by ${requiredDatapoints}.`)
+      }
+
+  }
+
+  const dataSetInstance = instances[algo][prefix]
+
+  if(dataSetInstance.flatFeaturesColLen === 0)
+  {
+    dataSetInstance.featureCols = getFeaturedKeys({trainingCols, findGroups, verticalOhlcv, scaledGroups})
+
+    if(areKeyValuesValid(main, index, dataSetInstance.featureCols))
+    {
+      dataSetInstance.flatFeaturesColLen = computeFlatFeaturesLen(dataSetInstance.featureCols, verticalOhlcv, index)
+
+      logMlTraining({
+        featureCols: dataSetInstance.featureCols, 
+        flatFeaturesColLen: dataSetInstance.flatFeaturesColLen, 
+        type, 
+        trainingSize
+      })
+    } else {
+      return
+    }
+  }
+
+  if(dataSetInstance.flatFeaturesColLen === 0) return
+
+  const {
+    uniqueLabels,
+    expectedLoops,
+    featureCols,
+    flatFeaturesColLen,
+    shouldRetrain,
+    X: xRows
+  } = dataSetInstance
+
+  const trainX = buildTrainX({
+    featureCols,
+    flatFeaturesColLen,
+    type,
+    index,
+    lookbackAbs,
+    main
+  })
+
+  if(!trainX) return 
+
+   //if univariable Y (flatY) a model is created for each prediction
+  const trainY = yCallback(index, verticalOhlcv, horizon)
+
+  if (trainY !== null && (!Array.isArray(trainY) || trainY.length !== predictions)) {
+
+    throw new Error(`trainY must be null or an array with ${predictions} items, got "${JSON.stringify(trainY)}" at index ${index} for ${type}`);
+  }
+
+  for(let loopIdx = 0; loopIdx < expectedLoops; loopIdx++)
+  {
+    const predictionKey = `${prefix}_${(loopIdx+1)}`
+    const yRows = dataSetInstance.Y[loopIdx]
+    const currTrainY =  (flatY) ? (trainY === null) ? null : trainY[loopIdx] : trainY
+    const isTrained = dataSetInstance.isTrained[loopIdx]
+
+    const shouldPredict = allModels.hasOwnProperty(predictionKey) && isTrained && (shouldRetrain === false || allModels[predictionKey].length === horizon)
+
+    //predicts using previously saved models even if current currTrainY is not available
+    if(shouldPredict)
+    {
+      const futureRow = allModels[predictionKey][0].predict([trainX])[0]
+
+      if(flatY)
+      {
+        if(futureRow == null) throw new Error(`Prediction of ${prefix} at index ${index} was expecting a number.`)
+
+        main.pushToMain({index, key: predictionKey, value: futureRow})
+
+        if(currTrainY != null)
+        {
+          updateClassifierMetrics({
+            metrics: ML.metrics[predictionKey], 
+            trueLabel: currTrainY, 
+            predictedLabel: futureRow
+          })
+        }
+      }
+      else {
+
+        if (!Array.isArray(futureRow) || futureRow.length !== predictions) {
+          throw new Error(
+            `Prediction output of ${algo} "${prefix}" at index ${index} was expecting an array of values.`
+          )
+        }
+
+        for(let preIdx = 0; preIdx < predictions; preIdx++) {
+          main.pushToMain({index, key: `${prefix}_${(preIdx+1)}`, value: futureRow[preIdx]})
+
+          if(Array.isArray(currTrainY) && currTrainY.length === predictions)
+          {
+            updateClassifierMetrics({
+              metrics: ML.metrics[`${prefix}_${(preIdx+1)}`], 
+              trueLabel: currTrainY[preIdx], 
+              predictedLabel: futureRow[preIdx]
+            })
+          }
+
+        }
+      }
+    }
+
+    if((index + predictions + 1) > len) continue 
+    if(currTrainY === null) continue // future not defined
+
+    if(flatY)
+    {
+        if (typeof currTrainY === 'undefined' || currTrainY == null)
+        {
+            throw new Error(`currTrainY must return number, string or boolean got ${typeof currTrainY} at index ${index}`)
+        }
+    } else {
+
+        if ((currTrainY.length !== predictions)) {
+
+          throw new Error(
+            `The number of label columns returned (${currTrainY.length}) doesn’t match the options.predictions (${predictions}) setting for the ${algo} "${prefix}" classifier. ` +
+            `Please update your yCallback function so its output array items align with options.predictions.`
+          );
+
+        }           
+    }
+
+    // enqueue
+    
+    yRows.push(currTrainY)
+    if (yRows.length > trainingSize) yRows.shift()
+
+    if(loopIdx === 0)
+    {
+      xRows.push(trainX)
+      if (xRows.length > trainingSize) xRows.shift()
+    }
+
+    const shouldTrainModel = shouldRetrain || !isTrained
+
+    if (shouldTrainModel && xRows.length === trainingSize && yRows.length === trainingSize) {
+
+      if(isTrained === false)
+      {
+        uniqueLabels[loopIdx] = countUniqueLabels(yRows)
+        
+        if(uniqueLabels[loopIdx] < 2)
+        {
+          throw new Error(`Invalid number or labels in ${prefix}. Check the logic of your "yCallback" function.`)
+        }
+      }
+
+
+      const {train} = modelConfig
+
+      console.log(yRows)
+
+      const trainedModel = train({modelClass, xRows, yRows, modelArgs, uniqueLabels: uniqueLabels[loopIdx]})
+
+      allModels[predictionKey].push(
+       trainedModel
+      )
+
+      if(allModels[predictionKey].length > horizon)
+      {
+        allModels[predictionKey].shift()
+      }
+
+      dataSetInstance.isTrained[loopIdx] = true
+    }
+  }
+
+  return true
+}

package/src/machine-learning/ml-config.js

@@ -0,0 +1,219 @@
+export const defaultYCallback = (index, verticalOhlcv, horizon) => {
+
+    //this function will be executed inside a secondary loop after most of the indicates are calculated
+    //use the "verticalOhlcv" object to access desired indicators
+
+    if(horizon !== 2) throw new Error(`The default "yCallback" function in this classifier requires that "options.horizon" and "options.predictions" are set to "2".`)
+
+    const currClose =  verticalOhlcv.close[index]
+    const nextCloseArr = Array.from({length: horizon}, (_, i) => verticalOhlcv.close[i + 1 + index])
+
+    if(nextCloseArr.some(v => typeof v === 'undefined')) return null
+
+    const [nextClose, nextNextClose] = nextCloseArr
+
+  
+    //the total length of item is this output must be equal to "options.yColumns" property
+    return [Number(nextClose > currClose), Number(nextNextClose > nextClose)]
+}
+
+export const exportTrainedModels = main => {
+
+  const {ML, inputParams} = main
+  const allModels = ML.models
+  const output = {}
+
+  for (const { key, params } of inputParams) {
+
+    if(key !== 'classifier' && key !== 'regressor') continue
+
+    const [_, {type, predictions, precompute}] = params
+    const {flatY, prefix} = precompute
+    const expectedLoops = (flatY) ? predictions : 1
+
+    let exportModel
+
+    if(key === 'regressor')
+    {
+      exportModel = validRegressors[type].exportModel
+    }
+    else if (key === 'classifier')
+    {
+      exportModel = validClassifiers[type].exportModel
+    }
+
+    for(let loopIdx = 0; loopIdx < expectedLoops; loopIdx++)
+    {
+      const modelKey = `${prefix}_${(loopIdx+1)}`
+      output[modelKey] = exportModel(allModels[modelKey])
+    }
+
+  }
+
+  return output
+}
+
+
+export const validClassifiers = {
+    'KNN': {
+      shortName: 'knn',
+      flatY: true,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs}) => new modelClass(xRows, yRows, modelArgs),
+      unsupervised: false
+    },
+    'FeedForwardNeuralNetworks': {
+      shortName: 'feed_forward',
+      flatY: false,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs, uniqueLabels}) => {
+
+        if(!modelArgs) modelArgs = {}
+
+        modelArgs = {
+          ...modelArgs, 
+          activation: (uniqueLabels === 2) ? 'logistic' : 'identity'
+        }
+        
+        const m = new modelClass(modelArgs)
+        m.train(xRows, yRows)
+        return m
+      },
+      unsupervised: false
+    },
+    'GaussianNB': {
+      shortName: 'naive_bayes',
+      flatY: true,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs}) => {
+        const m = new modelClass(modelArgs)
+        m.train(xRows, yRows)
+        return m
+      },
+      unsupervised: false
+    },
+    'MultinomialNB': {
+      shortName: 'naive_bayes',
+      flatY: true,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs}) => {
+        const m = new modelClass(modelArgs)
+        m.train(xRows, yRows)
+        return m
+      },
+      unsupervised: false
+    },
+    'DecisionTreeClassifier': {
+      shortName: 'decision_tree',
+      flatY: true,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs}) => {
+        const m = new modelClass(modelArgs)
+        m.train(xRows, yRows)
+        return m
+      },
+      unsupervised: false
+    },
+    'RandomForestClassifier': {
+      shortName: 'random_forest',
+      flatY: true,
+      exportModel: m => m.toJSON(),
+      train: ({modelClass, xRows, yRows, modelArgs}) => {
+        const m = new modelClass(modelArgs)
+        m.train(xRows, yRows)
+        return m
+      }
+    },
+    unsupervised: false
+}
+
+export const validRegressors = {
+    'ARIMA': {
+        shortName: 'arima',
+        flatX: true,
+        flatY: false,
+        exportModel: m => m,
+        train: ({modelClass, xRows, modelArgs}) => {
+
+          if(!modelArgs) modelArgs = {}
+
+          modelArgs = { ...modelArgs, p: 2, d: 1, q: 2, P: 0, D: 0, Q: 0, S: 0, verbose: false }
+
+          const m = new modelClass(modelArgs)
+          m.train(xRows)
+          return m
+        },
+        unsupervised: true
+    }, 
+    'SimpleLinearRegression': {
+        shortName: 'linear',
+        flatX: true,
+        flatY: true,
+  
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => new modelClass(xRows, yRows, modelArgs),
+        unsupervised: false
+    }, 
+    'PolynomialRegression': {
+        shortName: 'polynomial',
+        flatX: true,
+        flatY: true,
+  
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => new modelClass(xRows, yRows, modelArgs),
+        unsupervised: false
+    },
+    'MultivariateLinearRegression': {
+        shortName: 'multivariable',
+        flatX: false,
+        flatY: false,
+  
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => new modelClass(xRows, yRows, modelArgs),
+        unsupervised: false
+    }, 
+    'DecisionTreeRegression': {
+        shortName: 'decision_tree',
+        flatX: false,
+        flatY: true,
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => {
+          const m = new modelClass(modelArgs)
+          m.train(xRows, yRows)
+          return m
+        },
+        unsupervised: false
+    },
+    'RandomForestRegression': {
+        shortName: 'random_forest',
+        flatX: false,
+        flatY: true,
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => {
+          const m = new modelClass(modelArgs)
+          m.train(xRows, yRows)
+          return m
+        },
+        unsupervised: false
+    },
+    'FeedForwardNeuralNetworks': {
+        shortName: 'feed_forward',
+        flatX: false,
+        flatY: false,
+        exportModel: m => m.toJSON(),
+        train: ({modelClass, xRows, yRows, modelArgs}) => {
+
+          if(!modelArgs) modelArgs = {}
+
+          modelArgs = {
+            ...modelArgs, 
+            activation: 'identity'
+          }
+          
+          const m = new modelClass(modelArgs)
+          m.train(xRows, yRows)
+          return m
+        },
+        unsupervised: false
+    }
+}