bun-scikit 0.1.1

package/src/tree/DecisionTreeRegressor.ts

@@ -0,0 +1,407 @@
+import type { Matrix, RegressionModel, Vector } from "../types";
+import { r2Score } from "../metrics/regression";
+import {
+  assertConsistentRowSize,
+  assertFiniteMatrix,
+  assertFiniteVector,
+  validateRegressionInputs,
+} from "../utils/validation";
+import type { MaxFeaturesOption } from "./DecisionTreeClassifier";
+
+export interface DecisionTreeRegressorOptions {
+  maxDepth?: number;
+  minSamplesSplit?: number;
+  minSamplesLeaf?: number;
+  maxFeatures?: MaxFeaturesOption;
+  randomState?: number;
+}
+
+interface TreeNode {
+  prediction: number;
+  featureIndex?: number;
+  threshold?: number;
+  left?: TreeNode;
+  right?: TreeNode;
+  isLeaf: boolean;
+}
+
+interface SplitEvaluation {
+  threshold: number;
+  impurity: number;
+}
+
+interface SplitPartition {
+  leftIndices: number[];
+  rightIndices: number[];
+}
+
+const MAX_THRESHOLD_BINS = 128;
+
+function mulberry32(seed: number): () => number {
+  let state = seed >>> 0;
+  return () => {
+    state += 0x6d2b79f5;
+    let t = Math.imul(state ^ (state >>> 15), 1 | state);
+    t ^= t + Math.imul(t ^ (t >>> 7), 61 | t);
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+function safeVariance(sum: number, sumSquares: number, count: number): number {
+  if (count <= 0) {
+    return 0;
+  }
+  const mean = sum / count;
+  const variance = sumSquares / count - mean * mean;
+  return variance < 0 ? 0 : variance;
+}
+
+export class DecisionTreeRegressor implements RegressionModel {
+  private readonly maxDepth: number;
+  private readonly minSamplesSplit: number;
+  private readonly minSamplesLeaf: number;
+  private readonly maxFeatures: MaxFeaturesOption;
+  private readonly randomState?: number;
+  private random: () => number = Math.random;
+  private root: TreeNode | null = null;
+  private flattenedXTrain: Float64Array | null = null;
+  private yTrain: Float64Array | null = null;
+  private featureCount = 0;
+  private allFeatureIndices: number[] = [];
+  private featureSelectionMarks: Uint8Array | null = null;
+  private binTotals: Uint32Array = new Uint32Array(MAX_THRESHOLD_BINS);
+  private binSums: Float64Array = new Float64Array(MAX_THRESHOLD_BINS);
+  private binSumsSquares: Float64Array = new Float64Array(MAX_THRESHOLD_BINS);
+
+  constructor(options: DecisionTreeRegressorOptions = {}) {
+    this.maxDepth = options.maxDepth ?? 12;
+    this.minSamplesSplit = options.minSamplesSplit ?? 2;
+    this.minSamplesLeaf = options.minSamplesLeaf ?? 1;
+    this.maxFeatures = options.maxFeatures ?? null;
+    this.randomState = options.randomState;
+  }
+
+  fit(
+    X: Matrix,
+    y: Vector,
+    sampleIndices?: ArrayLike<number>,
+    skipValidation = false,
+    flattenedXTrain?: Float64Array,
+    yTrain?: Float64Array,
+  ): this {
+    if (!skipValidation) {
+      validateRegressionInputs(X, y);
+    }
+
+    this.featureCount = X[0].length;
+    this.flattenedXTrain = flattenedXTrain ?? this.flattenTrainingMatrix(X);
+    this.yTrain = yTrain ?? this.toFloat64Vector(y);
+    this.allFeatureIndices = new Array<number>(this.featureCount);
+    for (let i = 0; i < this.featureCount; i += 1) {
+      this.allFeatureIndices[i] = i;
+    }
+    this.featureSelectionMarks = new Uint8Array(this.featureCount);
+    this.random = this.randomState === undefined ? Math.random : mulberry32(this.randomState);
+
+    let rootIndices: number[];
+    if (sampleIndices) {
+      if (sampleIndices.length === 0) {
+        throw new Error("sampleIndices must not be empty.");
+      }
+      rootIndices = new Array<number>(sampleIndices.length);
+      for (let i = 0; i < sampleIndices.length; i += 1) {
+        const index = sampleIndices[i];
+        if (!Number.isInteger(index) || index < 0 || index >= X.length) {
+          throw new Error(`sampleIndices contains invalid index: ${index}.`);
+        }
+        rootIndices[i] = index;
+      }
+    } else {
+      rootIndices = new Array<number>(X.length);
+      for (let i = 0; i < X.length; i += 1) {
+        rootIndices[i] = i;
+      }
+    }
+
+    this.root = this.buildTree(rootIndices, 0);
+    return this;
+  }
+
+  predict(X: Matrix): Vector {
+    if (!this.root || this.featureCount === 0) {
+      throw new Error("DecisionTreeRegressor has not been fitted.");
+    }
+
+    assertConsistentRowSize(X);
+    assertFiniteMatrix(X);
+
+    if (X[0].length !== this.featureCount) {
+      throw new Error(
+        `Feature size mismatch. Expected ${this.featureCount}, got ${X[0].length}.`,
+      );
+    }
+
+    return X.map((sample) => this.predictOne(sample, this.root!));
+  }
+
+  score(X: Matrix, y: Vector): number {
+    assertFiniteVector(y);
+    return r2Score(y, this.predict(X));
+  }
+
+  private predictOne(sample: Vector, node: TreeNode): number {
+    let current = node;
+    while (
+      !current.isLeaf &&
+      current.featureIndex !== undefined &&
+      current.threshold !== undefined
+    ) {
+      if (sample[current.featureIndex] <= current.threshold) {
+        current = current.left!;
+      } else {
+        current = current.right!;
+      }
+    }
+    return current.prediction;
+  }
+
+  private buildTree(indices: number[], depth: number): TreeNode {
+    const y = this.yTrain!;
+    const sampleCount = indices.length;
+
+    let sum = 0;
+    let sumSquares = 0;
+    for (let i = 0; i < sampleCount; i += 1) {
+      const value = y[indices[i]];
+      sum += value;
+      sumSquares += value * value;
+    }
+    const prediction = sum / sampleCount;
+    const parentVariance = safeVariance(sum, sumSquares, sampleCount);
+
+    const depthStop = depth >= this.maxDepth;
+    const splitStop = sampleCount < this.minSamplesSplit;
+    const pureEnough = parentVariance <= 1e-14;
+    if (depthStop || splitStop || pureEnough) {
+      return { isLeaf: true, prediction };
+    }
+
+    const candidateFeatures = this.selectFeatureIndices(this.featureCount);
+    let bestFeature = -1;
+    let bestSplit: SplitEvaluation | null = null;
+
+    for (let i = 0; i < candidateFeatures.length; i += 1) {
+      const featureIndex = candidateFeatures[i];
+      const split = this.findBestThreshold(indices, featureIndex);
+      if (!split) {
+        continue;
+      }
+      if (!bestSplit || split.impurity < bestSplit.impurity) {
+        bestFeature = featureIndex;
+        bestSplit = split;
+      }
+    }
+
+    if (!bestSplit || bestFeature === -1 || bestSplit.impurity >= parentVariance - 1e-14) {
+      return { isLeaf: true, prediction };
+    }
+
+    const partition = this.partitionIndices(indices, bestFeature, bestSplit.threshold);
+    if (!partition) {
+      return { isLeaf: true, prediction };
+    }
+
+    return {
+      isLeaf: false,
+      prediction,
+      featureIndex: bestFeature,
+      threshold: bestSplit.threshold,
+      left: this.buildTree(partition.leftIndices, depth + 1),
+      right: this.buildTree(partition.rightIndices, depth + 1),
+    };
+  }
+
+  private resolveMaxFeatures(featureCount: number): number {
+    if (this.maxFeatures === null || this.maxFeatures === undefined) {
+      return featureCount;
+    }
+    if (this.maxFeatures === "sqrt") {
+      return Math.max(1, Math.floor(Math.sqrt(featureCount)));
+    }
+    if (this.maxFeatures === "log2") {
+      return Math.max(1, Math.floor(Math.log2(featureCount)));
+    }
+    return Math.max(1, Math.min(featureCount, Math.floor(this.maxFeatures)));
+  }
+
+  private selectFeatureIndices(featureCount: number): number[] {
+    const k = this.resolveMaxFeatures(featureCount);
+    if (k >= featureCount) {
+      return this.allFeatureIndices;
+    }
+
+    const marks = this.featureSelectionMarks!;
+    marks.fill(0);
+    const selected = new Array<number>(k);
+    let count = 0;
+    while (count < k) {
+      const candidate = Math.floor(this.random() * featureCount);
+      if (marks[candidate] !== 0) {
+        continue;
+      }
+      marks[candidate] = 1;
+      selected[count] = candidate;
+      count += 1;
+    }
+    return selected;
+  }
+
+  private findBestThreshold(indices: number[], featureIndex: number): SplitEvaluation | null {
+    const x = this.flattenedXTrain!;
+    const y = this.yTrain!;
+    const stride = this.featureCount;
+    const sampleCount = indices.length;
+
+    let minValue = Number.POSITIVE_INFINITY;
+    let maxValue = Number.NEGATIVE_INFINITY;
+    let totalSum = 0;
+    let totalSumSquares = 0;
+
+    for (let i = 0; i < sampleCount; i += 1) {
+      const sampleIndex = indices[i];
+      const xValue = x[sampleIndex * stride + featureIndex];
+      const yValue = y[sampleIndex];
+      if (xValue < minValue) {
+        minValue = xValue;
+      }
+      if (xValue > maxValue) {
+        maxValue = xValue;
+      }
+      totalSum += yValue;
+      totalSumSquares += yValue * yValue;
+    }
+
+    if (!Number.isFinite(minValue) || !Number.isFinite(maxValue) || minValue === maxValue) {
+      return null;
+    }
+
+    const dynamicBins = Math.floor(Math.sqrt(sampleCount));
+    const binCount = Math.max(16, Math.min(MAX_THRESHOLD_BINS, dynamicBins));
+    const binTotals = this.binTotals;
+    const binSums = this.binSums;
+    const binSumsSquares = this.binSumsSquares;
+    binTotals.fill(0, 0, binCount);
+    binSums.fill(0, 0, binCount);
+    binSumsSquares.fill(0, 0, binCount);
+
+    const range = maxValue - minValue;
+    for (let i = 0; i < sampleCount; i += 1) {
+      const sampleIndex = indices[i];
+      const xValue = x[sampleIndex * stride + featureIndex];
+      const yValue = y[sampleIndex];
+      let bin = Math.floor(((xValue - minValue) / range) * binCount);
+      if (bin < 0) {
+        bin = 0;
+      } else if (bin >= binCount) {
+        bin = binCount - 1;
+      }
+      binTotals[bin] += 1;
+      binSums[bin] += yValue;
+      binSumsSquares[bin] += yValue * yValue;
+    }
+
+    let leftCount = 0;
+    let leftSum = 0;
+    let leftSumSquares = 0;
+    let bestImpurity = Number.POSITIVE_INFINITY;
+    let bestThreshold = 0;
+
+    for (let bin = 0; bin < binCount - 1; bin += 1) {
+      leftCount += binTotals[bin];
+      leftSum += binSums[bin];
+      leftSumSquares += binSumsSquares[bin];
+
+      const rightCount = sampleCount - leftCount;
+      if (leftCount < this.minSamplesLeaf || rightCount < this.minSamplesLeaf) {
+        continue;
+      }
+
+      const rightSum = totalSum - leftSum;
+      const rightSumSquares = totalSumSquares - leftSumSquares;
+
+      const leftVariance = safeVariance(leftSum, leftSumSquares, leftCount);
+      const rightVariance = safeVariance(rightSum, rightSumSquares, rightCount);
+      const impurity =
+        (leftCount / sampleCount) * leftVariance + (rightCount / sampleCount) * rightVariance;
+
+      if (impurity < bestImpurity) {
+        bestImpurity = impurity;
+        bestThreshold = minValue + (range * (bin + 1)) / binCount;
+      }
+    }
+
+    if (!Number.isFinite(bestImpurity)) {
+      return null;
+    }
+
+    return {
+      threshold: bestThreshold,
+      impurity: bestImpurity,
+    };
+  }
+
+  private partitionIndices(
+    indices: number[],
+    featureIndex: number,
+    threshold: number,
+  ): SplitPartition | null {
+    const x = this.flattenedXTrain!;
+    const stride = this.featureCount;
+    const sampleCount = indices.length;
+    const leftIndices = new Array<number>(sampleCount);
+    const rightIndices = new Array<number>(sampleCount);
+    let leftCount = 0;
+    let rightCount = 0;
+
+    for (let i = 0; i < sampleCount; i += 1) {
+      const sampleIndex = indices[i];
+      if (x[sampleIndex * stride + featureIndex] <= threshold) {
+        leftIndices[leftCount] = sampleIndex;
+        leftCount += 1;
+      } else {
+        rightIndices[rightCount] = sampleIndex;
+        rightCount += 1;
+      }
+    }
+
+    if (leftCount < this.minSamplesLeaf || rightCount < this.minSamplesLeaf) {
+      return null;
+    }
+
+    return {
+      leftIndices: leftIndices.slice(0, leftCount),
+      rightIndices: rightIndices.slice(0, rightCount),
+    };
+  }
+
+  private flattenTrainingMatrix(X: Matrix): Float64Array {
+    const sampleCount = X.length;
+    const flattened = new Float64Array(sampleCount * this.featureCount);
+    for (let i = 0; i < sampleCount; i += 1) {
+      const row = X[i];
+      const rowOffset = i * this.featureCount;
+      for (let j = 0; j < this.featureCount; j += 1) {
+        flattened[rowOffset + j] = row[j];
+      }
+    }
+    return flattened;
+  }
+
+  private toFloat64Vector(y: Vector): Float64Array {
+    const out = new Float64Array(y.length);
+    for (let i = 0; i < y.length; i += 1) {
+      out[i] = y[i];
+    }
+    return out;
+  }
+}

package/src/types.ts

@@ -0,0 +1,18 @@
+export type Vector = number[];
+export type Matrix = number[][];
+
+export interface Transformer {
+  fit(X: Matrix, y?: Vector): this;
+  transform(X: Matrix): Matrix;
+  fitTransform?(X: Matrix, y?: Vector): Matrix;
+}
+
+export interface RegressionModel {
+  fit(X: Matrix, y: Vector): this;
+  predict(X: Matrix): Vector;
+}
+
+export interface ClassificationModel {
+  fit(X: Matrix, y: Vector): this;
+  predict(X: Matrix): Vector;
+}

package/src/utils/linalg.ts

@@ -0,0 +1,209 @@
+import type { Matrix, Vector } from "../types";
+
+export function transpose(X: Matrix): Matrix {
+  const rows = X.length;
+  const cols = X[0].length;
+  const result: Matrix = Array.from({ length: cols }, () =>
+    new Array(rows).fill(0),
+  );
+
+  for (let i = 0; i < rows; i += 1) {
+    for (let j = 0; j < cols; j += 1) {
+      result[j][i] = X[i][j];
+    }
+  }
+
+  return result;
+}
+
+export function multiplyMatrices(A: Matrix, B: Matrix): Matrix {
+  const aRows = A.length;
+  const aCols = A[0].length;
+  const bRows = B.length;
+  const bCols = B[0].length;
+
+  if (aCols !== bRows) {
+    throw new Error(
+      `Matrix dimensions do not align: ${aRows}x${aCols} times ${bRows}x${bCols}.`,
+    );
+  }
+
+  const result: Matrix = Array.from({ length: aRows }, () =>
+    new Array(bCols).fill(0),
+  );
+
+  for (let i = 0; i < aRows; i += 1) {
+    for (let k = 0; k < aCols; k += 1) {
+      const aik = A[i][k];
+      for (let j = 0; j < bCols; j += 1) {
+        result[i][j] += aik * B[k][j];
+      }
+    }
+  }
+
+  return result;
+}
+
+export function multiplyMatrixVector(A: Matrix, x: Vector): Vector {
+  const rows = A.length;
+  const cols = A[0].length;
+
+  if (cols !== x.length) {
+    throw new Error(
+      `Matrix-vector dimensions do not align: ${rows}x${cols} times ${x.length}.`,
+    );
+  }
+
+  const result = new Array(rows).fill(0);
+  for (let i = 0; i < rows; i += 1) {
+    let sum = 0;
+    for (let j = 0; j < cols; j += 1) {
+      sum += A[i][j] * x[j];
+    }
+    result[i] = sum;
+  }
+
+  return result;
+}
+
+export function addInterceptColumn(X: Matrix): Matrix {
+  return X.map((row) => [1, ...row]);
+}
+
+export function identityMatrix(size: number): Matrix {
+  const I: Matrix = Array.from({ length: size }, () => new Array(size).fill(0));
+  for (let i = 0; i < size; i += 1) {
+    I[i][i] = 1;
+  }
+  return I;
+}
+
+export function inverseMatrix(A: Matrix): Matrix {
+  const n = A.length;
+  if (n === 0 || A[0].length !== n) {
+    throw new Error("Only non-empty square matrices can be inverted.");
+  }
+
+  const EPSILON = 1e-12;
+  const augmented: Matrix = A.map((row, i) => [...row, ...identityMatrix(n)[i]]);
+
+  for (let col = 0; col < n; col += 1) {
+    let pivotRow = col;
+    let maxAbs = Math.abs(augmented[pivotRow][col]);
+
+    for (let r = col + 1; r < n; r += 1) {
+      const value = Math.abs(augmented[r][col]);
+      if (value > maxAbs) {
+        maxAbs = value;
+        pivotRow = r;
+      }
+    }
+
+    if (maxAbs < EPSILON) {
+      throw new Error("Matrix is singular or near-singular and cannot be inverted.");
+    }
+
+    if (pivotRow !== col) {
+      const tmp = augmented[col];
+      augmented[col] = augmented[pivotRow];
+      augmented[pivotRow] = tmp;
+    }
+
+    const pivot = augmented[col][col];
+    for (let j = 0; j < 2 * n; j += 1) {
+      augmented[col][j] /= pivot;
+    }
+
+    for (let r = 0; r < n; r += 1) {
+      if (r === col) {
+        continue;
+      }
+
+      const factor = augmented[r][col];
+      if (factor === 0) {
+        continue;
+      }
+
+      for (let j = 0; j < 2 * n; j += 1) {
+        augmented[r][j] -= factor * augmented[col][j];
+      }
+    }
+  }
+
+  return augmented.map((row) => row.slice(n));
+}
+
+export function solveSymmetricPositiveDefinite(A: Matrix, b: Vector): Vector {
+  const n = A.length;
+  if (n === 0 || A[0].length !== n) {
+    throw new Error("A must be a non-empty square matrix.");
+  }
+  if (b.length !== n) {
+    throw new Error(`b length must match matrix size ${n}. Got ${b.length}.`);
+  }
+
+  const L: Matrix = Array.from({ length: n }, () => new Array(n).fill(0));
+  const EPSILON = 1e-12;
+
+  for (let i = 0; i < n; i += 1) {
+    for (let j = 0; j <= i; j += 1) {
+      let sum = A[i][j];
+      for (let k = 0; k < j; k += 1) {
+        sum -= L[i][k] * L[j][k];
+      }
+
+      if (i === j) {
+        if (sum <= EPSILON) {
+          throw new Error("Matrix is not positive definite.");
+        }
+        L[i][j] = Math.sqrt(sum);
+      } else {
+        L[i][j] = sum / L[j][j];
+      }
+    }
+  }
+
+  const y = new Array(n).fill(0);
+  for (let i = 0; i < n; i += 1) {
+    let sum = b[i];
+    for (let k = 0; k < i; k += 1) {
+      sum -= L[i][k] * y[k];
+    }
+    y[i] = sum / L[i][i];
+  }
+
+  const x = new Array(n).fill(0);
+  for (let i = n - 1; i >= 0; i -= 1) {
+    let sum = y[i];
+    for (let k = i + 1; k < n; k += 1) {
+      sum -= L[k][i] * x[k];
+    }
+    x[i] = sum / L[i][i];
+  }
+
+  return x;
+}
+
+export function dot(a: Vector, b: Vector): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector sizes do not match: ${a.length} vs ${b.length}.`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i += 1) {
+    sum += a[i] * b[i];
+  }
+  return sum;
+}
+
+export function mean(values: Vector): number {
+  if (values.length === 0) {
+    throw new Error("Cannot compute mean of an empty vector.");
+  }
+
+  let total = 0;
+  for (let i = 0; i < values.length; i += 1) {
+    total += values[i];
+  }
+  return total / values.length;
+}

package/src/utils/validation.ts

@@ -0,0 +1,78 @@
+import type { Matrix, Vector } from "../types";
+
+export function assertNonEmptyMatrix(X: Matrix, label = "X"): void {
+  if (!Array.isArray(X) || X.length === 0) {
+    throw new Error(`${label} must be a non-empty 2D array.`);
+  }
+
+  if (!Array.isArray(X[0]) || X[0].length === 0) {
+    throw new Error(`${label} must have at least one feature column.`);
+  }
+}
+
+export function assertConsistentRowSize(X: Matrix, label = "X"): void {
+  const featureCount = X[0]?.length ?? 0;
+  for (let rowIndex = 0; rowIndex < X.length; rowIndex += 1) {
+    const row = X[rowIndex];
+    if (!Array.isArray(row) || row.length !== featureCount) {
+      throw new Error(
+        `${label} rows must all have the same length. Row ${rowIndex} differs.`,
+      );
+    }
+  }
+}
+
+export function assertFiniteMatrix(X: Matrix, label = "X"): void {
+  for (let i = 0; i < X.length; i += 1) {
+    for (let j = 0; j < X[i].length; j += 1) {
+      const value = X[i][j];
+      if (!Number.isFinite(value)) {
+        throw new Error(`${label} contains a non-finite value at [${i}, ${j}].`);
+      }
+    }
+  }
+}
+
+export function assertVectorLength(
+  y: Vector,
+  expectedLength: number,
+  label = "y",
+): void {
+  if (!Array.isArray(y) || y.length !== expectedLength) {
+    throw new Error(`${label} length must equal ${expectedLength}.`);
+  }
+}
+
+export function assertFiniteVector(y: Vector, label = "y"): void {
+  for (let i = 0; i < y.length; i += 1) {
+    if (!Number.isFinite(y[i])) {
+      throw new Error(`${label} contains a non-finite value at index ${i}.`);
+    }
+  }
+}
+
+export function validateRegressionInputs(X: Matrix, y: Vector): void {
+  assertNonEmptyMatrix(X);
+  assertConsistentRowSize(X);
+  assertFiniteMatrix(X);
+  assertVectorLength(y, X.length);
+  assertFiniteVector(y);
+}
+
+export function assertBinaryVector(y: Vector, label = "y"): void {
+  for (let i = 0; i < y.length; i += 1) {
+    const value = y[i];
+    if (!(value === 0 || value === 1)) {
+      throw new Error(`${label} must be binary (0 or 1). Found ${value} at index ${i}.`);
+    }
+  }
+}
+
+export function validateClassificationInputs(X: Matrix, y: Vector): void {
+  assertNonEmptyMatrix(X);
+  assertConsistentRowSize(X);
+  assertFiniteMatrix(X);
+  assertVectorLength(y, X.length);
+  assertFiniteVector(y);
+  assertBinaryVector(y);
+}