easy_ml 0.1.4 → 0.2.0.pre.rc1

data/app/frontend/components/dataset/ColumnFilters.tsx

@@ -0,0 +1,256 @@
+import React, { useState } from 'react';
+import { Filter, Database, Wrench, Eye, EyeOff, AlertTriangle, ChevronLeft, ChevronRight } from 'lucide-react';
+import type { Column } from '../../types';
+
+const ITEMS_PER_PAGE = 5;
+interface ColumnFiltersProps {
+  types: string[];
+  activeFilters: {
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    types: string[];
+  };
+  onFilterChange: (filters: {
+    view: 'all' | 'training' | 'hidden' | 'preprocessed' | 'nulls';
+    types: string[];
+  }) => void;
+  columnStats: {
+    total: number;
+    filtered: number;
+    training: number;
+    hidden: number;
+    withPreprocessing: number;
+    withNulls: number;
+  };
+  colHasPreprocessingSteps: (col: Column) => boolean;
+  columns: Column[];
+}
+
+export function ColumnFilters({
+  types,
+  activeFilters,
+  onFilterChange,
+  columnStats,
+  colHasPreprocessingSteps,
+  columns
+}: ColumnFiltersProps) {
+  const getViewStats = (view: typeof activeFilters.view) => {
+    switch (view) {
+      case 'training':
+        return `${columnStats.training} columns`;
+      case 'hidden':
+        return `${columnStats.hidden} columns`;
+      case 'preprocessed':
+        return `${columnStats.withPreprocessing} columns`;
+      case 'nulls':
+        return `${columnStats.withNulls} columns`;
+      default:
+        return `${columnStats.total} columns`;
+    }
+  };
+
+  const calculateNullPercentage = (column: Column) => {
+    if (!column.statistics?.processed?.null_count || !column.statistics?.processed?.num_rows) return 0;
+    return (column.statistics.processed.null_count / column.statistics.processed.num_rows) * 100;
+  };
+
+  const columnsWithNulls = columns
+    .filter(col => col.statistics?.processed.null_count && col.statistics.processed.null_count > 0)
+    .sort((a, b) => calculateNullPercentage(b) - calculateNullPercentage(a));
+
+  const [currentPage, setCurrentPage] = useState(1);
+  const totalPages = Math.ceil(columnsWithNulls.length / ITEMS_PER_PAGE);
+  const paginatedColumns = columnsWithNulls.slice(
+    (currentPage - 1) * ITEMS_PER_PAGE,
+    currentPage * ITEMS_PER_PAGE
+  );
+
+  const toggleType = (type: string) => {
+    onFilterChange({
+      ...activeFilters,
+      types: activeFilters.types.includes(type)
+        ? activeFilters.types.filter(t => t !== type)
+        : [...activeFilters.types, type]
+    });
+  };
+
+  return (
+    <div className="p-4 border-b space-y-4">
+      <div className="flex items-center justify-between">
+        <h3 className="text-sm font-medium text-gray-900 flex items-center gap-2">
+          <Filter className="w-4 h-4" />
+          Column Views
+        </h3>
+        <div className="text-sm text-gray-500">
+          Showing {columnStats.filtered} of {columnStats.total} columns
+        </div>
+      </div>
+
+      <div className="space-y-4">
+        {/* View Selector */}
+        <div className="flex flex-wrap gap-2">
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'all' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'all'
+                ? 'bg-gray-100 text-gray-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Database className="w-4 h-4" />
+            All
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('all')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'training' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'training'
+                ? 'bg-green-100 text-green-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Eye className="w-4 h-4" />
+            Training
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('training')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'hidden' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'hidden'
+                ? 'bg-gray-100 text-gray-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <EyeOff className="w-4 h-4" />
+            Hidden
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('hidden')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'preprocessed' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'preprocessed'
+                ? 'bg-blue-100 text-blue-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <Wrench className="w-4 h-4" />
+            Preprocessed
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('preprocessed')})
+            </span>
+          </button>
+          <button
+            onClick={() => onFilterChange({ ...activeFilters, view: 'nulls' })}
+            className={`inline-flex items-center gap-1 px-3 py-1.5 rounded-md text-sm font-medium ${
+              activeFilters.view === 'nulls'
+                ? 'bg-yellow-100 text-yellow-900'
+                : 'text-gray-600 hover:bg-gray-50'
+            }`}
+          >
+            <AlertTriangle className="w-4 h-4" />
+            Has Nulls
+            <span className="text-xs text-gray-500 ml-1">
+              ({getViewStats('nulls')})
+            </span>
+          </button>
+        </div>
+
+        {/* Column Types */}
+        <div>
+          <label className="text-xs font-medium text-gray-700 mb-2 block">
+            Column Types
+          </label>
+          <div className="flex flex-wrap gap-2">
+            {types.map(type => (
+              <button
+                key={type}
+                onClick={() => toggleType(type)}
+                className={`px-2 py-1 rounded-md text-xs font-medium ${
+                  activeFilters.types.includes(type)
+                    ? 'bg-blue-100 text-blue-700'
+                    : 'bg-gray-100 text-gray-600 hover:bg-gray-200'
+                }`}
+              >
+                {type}
+              </button>
+            ))}
+          </div>
+        </div>
+
+        {activeFilters.view === 'preprocessed' && columnStats.withPreprocessing > 0 && (
+          <div className="bg-blue-50 rounded-lg p-3">
+            <h4 className="text-sm font-medium text-blue-900 mb-2">Preprocessing Overview</h4>
+            <div className="space-y-2">
+              {columns
+                .filter(colHasPreprocessingSteps)
+                .map(col => (
+                  <div key={col.name} className="flex items-center justify-between text-sm">
+                    <span className="text-blue-800">{col.name}</span>
+                    <span className="text-blue-600">
+                      {col.preprocessing_steps?.training.method}
+                    </span>
+                  </div>
+                ))}
+            </div>
+          </div>
+        )}
+
+        {activeFilters.view === 'nulls' && columnsWithNulls.length > 0 && (
+          <div className="bg-yellow-50 rounded-lg p-3">
+            <div className="flex items-center justify-between mb-3">
+              <h4 className="text-sm font-medium text-yellow-900">Null Value Distribution</h4>
+              <div className="flex items-center gap-2 text-sm text-yellow-700">
+                <button
+                  onClick={() => setCurrentPage(p => Math.max(1, p - 1))}
+                  disabled={currentPage === 1}
+                  className="p-1 rounded hover:bg-yellow-100 disabled:opacity-50"
+                >
+                  <ChevronLeft className="w-4 h-4" />
+                </button>
+                <span>
+                  Page {currentPage} of {totalPages}
+                </span>
+                <button
+                  onClick={() => setCurrentPage(p => Math.min(totalPages, p + 1))}
+                  disabled={currentPage === totalPages}
+                  className="p-1 rounded hover:bg-yellow-100 disabled:opacity-50"
+                >
+                  <ChevronRight className="w-4 h-4" />
+                </button>
+              </div>
+            </div>
+            <div className="space-y-2">
+              {paginatedColumns.map(col => (
+                <div key={col.name} className="flex items-center gap-2">
+                  <span className="text-yellow-800 text-sm min-w-[120px]">{col.name}</span>
+                  <div className="flex-1 h-2 bg-yellow-100 rounded-full overflow-hidden">
+                    <div
+                      className="h-full bg-yellow-400 rounded-full"
+                      style={{ width: `${calculateNullPercentage(col)}%` }}
+                    />
+                  </div>
+                  <div className="flex items-center gap-2">
+                    <span className="text-yellow-800 text-xs">
+                      {calculateNullPercentage(col).toFixed(1)}% null
+                    </span>
+                    <span className="text-yellow-600 text-xs">
+                      ({col.statistics?.nullCount?.toLocaleString()} / {col.statistics?.rowCount?.toLocaleString()})
+                    </span>
+                  </div>
+                </div>
+              ))}
+            </div>
+            <div className="mt-3 text-sm text-yellow-700">
+              {columnsWithNulls.length} columns contain null values
+            </div>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+}

data/app/frontend/components/dataset/ColumnList.tsx

@@ -0,0 +1,101 @@
+import React from 'react';
+import { Settings2, AlertCircle, Target, EyeOff, Eye } from 'lucide-react';
+import type { Column } from '../../types';
+import { usePage } from "@inertiajs/react";
+
+interface ColumnListProps {
+  columns: Column[];
+  selectedColumn: string | null;
+  onColumnSelect: (columnName: string) => void;
+  onToggleHidden: (columnName: string) => void;
+}
+
+export function ColumnList({
+  columns,
+  selectedColumn,
+  onColumnSelect,
+  onToggleHidden
+}: ColumnListProps) {
+  const { rootPath } = usePage().props;
+
+  return (
+    <div className="space-y-2 pb-2">
+      {columns.map(column => (
+        <div
+          key={column.name}
+          className={`p-3 rounded-lg border ${
+            selectedColumn === column.name
+              ? 'border-blue-500 bg-blue-50'
+              : column.is_target
+              ? 'border-purple-500 bg-purple-50'
+              : column.hidden
+              ? 'border-gray-200 bg-gray-50'
+              : 'border-gray-200 hover:border-gray-300'
+          } transition-colors duration-150`}
+        >
+          <div className="flex items-center justify-between mb-2">
+            <div className="flex items-center gap-2">
+              {column.is_target && (
+                <Target className="w-4 h-4 text-purple-500" />
+              )}
+              <span className={`font-medium ${column.hidden ? 'text-gray-500' : 'text-gray-900'}`}>
+                {column.name}
+              </span>
+              <span className="text-xs px-2 py-0.5 bg-gray-100 text-gray-600 rounded-full">
+                {column.datatype}
+              </span>
+            </div>
+            <div className="flex items-center gap-2">
+              {!column.is_target && (
+                <button
+                  onClick={() => onToggleHidden(column.name)}
+                  className={`p-1 rounded hover:bg-gray-100 ${
+                    column.hidden
+                      ? 'text-gray-500'
+                      : 'text-gray-400 hover:text-gray-600'
+                  }`}
+                  title={column.hidden ? 'Show column' : 'Hide column'}
+                >
+                  {column.hidden ? (
+                    <EyeOff className="w-4 h-4" />
+                  ) : (
+                    <Eye className="w-4 h-4" />
+                  )}
+                </button>
+              )}
+              <button
+                onClick={() => onColumnSelect(column.name)}
+                className="p-1 rounded text-gray-400 hover:text-gray-600 hover:bg-gray-100"
+                title="Configure preprocessing"
+              >
+                <Settings2 className="w-4 h-4" />
+              </button>
+            </div>
+          </div>
+          <div className="text-sm text-gray-500">
+            {column.description && (
+              <p className={`mb-1 line-clamp-1 ${column.drop_if_null ? 'text-gray-400' : ''}`}>
+                {column.description}
+              </p>
+            )}
+            <div className="flex flex-wrap gap-2">
+              {column.preprocessing_steps && column.preprocessing_steps?.training &&
+               column.preprocessing_steps?.training?.method !== 'none' && (
+                <div className="flex items-center gap-1 text-blue-600">
+                  <AlertCircle className="w-3 h-3" />
+                  <span className="text-xs">Preprocessing configured</span>
+                </div>
+              )}
+              {column.hidden && (
+                <div className="flex items-center gap-1 text-gray-400">
+                  <EyeOff className="w-3 h-3" />
+                  <span className="text-xs">Hidden from training</span>
+                </div>
+              )}
+            </div>
+          </div>
+        </div>
+      ))}
+    </div>
+  );
+}

data/app/frontend/components/dataset/FeatureConfigPopover.tsx

@@ -0,0 +1,57 @@
+import React from "react";
+import { Settings2 } from "lucide-react";
+import { Popover } from "../Popover";
+
+export function FeatureConfigPopover() {
+  return (
+    <Popover
+      trigger={
+        <button
+          type="button"
+          className="p-2 text-gray-400 hover:text-gray-600"
+          title="Configure features"
+        >
+          <Settings2 className="w-5 h-5" />
+        </button>
+      }
+      className="w-96"
+    >
+      <div className="space-y-4">
+        <p className="text-sm text-gray-600">
+          Feature options can be configured in the codebase, and loaded in
+          initializers:
+        </p>
+
+        <div className="bg-gray-50 p-3 rounded-md">
+          <code className="text-sm text-gray-800">
+            config/initializers/features.rb
+          </code>
+        </div>
+
+        <p className="text-sm text-gray-600">Example feature implementation:</p>
+
+        <pre className="bg-gray-50 p-3 rounded-md overflow-x-auto">
+          <code className="text-xs text-gray-800">
+            {`# lib/features/did_convert.rb
+module Features
+  class DidConvert
+    include EasyML::Features
+
+    def did_convert(df)
+      df.with_column(
+        (Polars.col("rev") > 0)
+          .alias("did_convert")
+      )
+    end
+    
+    feature :did_convert,
+      name: "Did Convert",
+      description: "Boolean true/false..."
+  end
+end`}
+          </code>
+        </pre>
+      </div>
+    </Popover>
+  );
+}

data/app/frontend/components/dataset/FeaturePicker.tsx

@@ -0,0 +1,205 @@
+import React, { useState } from "react";
+import {
+  GripVertical,
+  X,
+  Plus,
+  ArrowDown,
+  ArrowUp,
+  Settings2,
+} from "lucide-react";
+import { SearchableSelect } from "../SearchableSelect";
+import { FeatureConfigPopover } from "./FeatureConfigPopover";
+import { Feature } from "../../types/dataset";
+
+interface FeaturePickerProps {
+  options: Feature[];
+  initialFeatures?: Feature[];
+  onFeaturesChange: (features: Feature[]) => void;
+}
+
+export function FeaturePicker({
+  options,
+  initialFeatures = [],
+  onFeaturesChange,
+}: FeaturePickerProps) {
+  const [selectedFeatures, setSelectedFeatures] =
+    useState<Feature[]>(initialFeatures);
+  const [draggedIndex, setDraggedIndex] = useState<number | null>(null);
+
+  console.log(selectedFeatures);
+  const availableFeatures = options.filter(
+    (feature) => !selectedFeatures.find((t) => t.name === feature.name)
+  );
+
+  const updateFeatures = (newFeatures: Feature[]) => {
+    const featuresWithPosition = newFeatures.map((feature, index) => ({
+      ...feature,
+      feature_position: index,
+    }));
+
+    setSelectedFeatures(featuresWithPosition);
+    onFeaturesChange(featuresWithPosition);
+  };
+
+  const handleAddFeature = (transformName: string) => {
+    const feature = options.find((t) => t.name === transformName);
+    if (feature) {
+      const newFeature = {
+        ...feature,
+        feature_position: selectedFeatures.length,
+      };
+      updateFeatures([...selectedFeatures, newFeature]);
+    }
+  };
+
+  const handleRemove = (index: number) => {
+    const newFeatures = [...selectedFeatures];
+    newFeatures.splice(index, 1);
+    updateFeatures(newFeatures);
+  };
+
+  const handleMoveUp = (index: number) => {
+    if (index === 0) return;
+    const newFeatures = [...selectedFeatures];
+    [newFeatures[index - 1], newFeatures[index]] = [
+      newFeatures[index],
+      newFeatures[index - 1],
+    ];
+    updateFeatures(newFeatures);
+  };
+
+  const handleMoveDown = (index: number) => {
+    if (index === selectedFeatures.length - 1) return;
+    const newFeatures = [...selectedFeatures];
+    [newFeatures[index], newFeatures[index + 1]] = [
+      newFeatures[index + 1],
+      newFeatures[index],
+    ];
+    updateFeatures(newFeatures);
+  };
+
+  const handleDragStart = (e: React.DragEvent, index: number) => {
+    setDraggedIndex(index);
+  };
+
+  const handleDragOver = (e: React.DragEvent, index: number) => {
+    e.preventDefault();
+    if (draggedIndex === null || draggedIndex === index) return;
+
+    const newFeatures = [...selectedFeatures];
+    const [draggedFeature] = newFeatures.splice(draggedIndex, 1);
+    newFeatures.splice(index, 0, draggedFeature);
+    updateFeatures(newFeatures);
+    setDraggedIndex(index);
+  };
+
+  const handleDragEnd = () => {
+    setDraggedIndex(null);
+  };
+
+  return (
+    <div className="space-y-4">
+      {/* Add Feature */}
+      <div className="flex items-center gap-4">
+        <div className="flex-1">
+          <SearchableSelect
+            options={availableFeatures.map((feature) => ({
+              value: feature.name,
+              label: feature.name,
+              description: feature.description,
+            }))}
+            value=""
+            onChange={(value) => handleAddFeature(value as string)}
+            placeholder="Add a transform..."
+          />
+        </div>
+        <FeatureConfigPopover />
+      </div>
+
+      {/* Selected Features */}
+      <div className="space-y-2">
+        {selectedFeatures.map((feature, index) => (
+          <div
+            key={feature.name}
+            draggable
+            onDragStart={(e) => handleDragStart(e, index)}
+            onDragOver={(e) => handleDragOver(e, index)}
+            onDragEnd={handleDragEnd}
+            className={`flex items-center gap-3 p-3 bg-white border rounded-lg ${
+              draggedIndex === index
+                ? "border-blue-500 shadow-lg"
+                : "border-gray-200"
+            } ${draggedIndex !== null ? "cursor-grabbing" : ""}`}
+          >
+            <button
+              type="button"
+              className="p-1 text-gray-400 hover:text-gray-600 cursor-grab active:cursor-grabbing"
+            >
+              <GripVertical className="w-4 h-4" />
+            </button>
+
+            <div className="flex-1 min-w-0">
+              <div className="flex items-center gap-2">
+                <span className="font-medium text-gray-900">
+                  {feature.name}
+                </span>
+                <span
+                  className={`text-xs px-2 py-0.5 rounded-full ${
+                    feature.feature_type === "calculation"
+                      ? "bg-blue-100 text-blue-800"
+                      : feature.feature_type === "lookup"
+                      ? "bg-purple-100 text-purple-800"
+                      : "bg-green-100 text-green-800"
+                  }`}
+                >
+                  {"feature"}
+                </span>
+              </div>
+              <p className="text-sm text-gray-500 truncate">
+                {feature.description}
+              </p>
+            </div>
+
+            <div className="flex items-center gap-1">
+              <button
+                type="button"
+                onClick={() => handleMoveUp(index)}
+                disabled={index === 0}
+                className="p-1 text-gray-400 hover:text-gray-600 disabled:opacity-50"
+                title="Move up"
+              >
+                <ArrowUp className="w-4 h-4" />
+              </button>
+              <button
+                type="button"
+                onClick={() => handleMoveDown(index)}
+                disabled={index === selectedFeatures.length - 1}
+                className="p-1 text-gray-400 hover:text-gray-600 disabled:opacity-50"
+                title="Move down"
+              >
+                <ArrowDown className="w-4 h-4" />
+              </button>
+              <button
+                type="button"
+                onClick={() => handleRemove(index)}
+                className="p-1 text-gray-400 hover:text-red-600"
+                title="Remove transform"
+              >
+                <X className="w-4 h-4" />
+              </button>
+            </div>
+          </div>
+        ))}
+
+        {selectedFeatures.length === 0 && (
+          <div className="text-center py-8 bg-gray-50 border-2 border-dashed border-gray-200 rounded-lg">
+            <Plus className="w-8 h-8 text-gray-400 mx-auto mb-2" />
+            <p className="text-sm text-gray-500">
+              Add features to enrich your dataset
+            </p>
+          </div>
+        )}
+      </div>
+    </div>
+  );
+}