j1-template 2022.5.0.rc1 → 2022.5.0.rc4

data/lib/starter_web/pages/public/_jupyter/notebooks/martin_renou_bqplot/Applications/Outlier Detection.ipynb

@@ -0,0 +1,311 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Outlier Detection with `bqplot`\n",
+    "---"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "In this notebook, we create a class `DNA` that leverages the new bqplot canvas based [HeatMap](https://github.com/bloomberg/bqplot/blob/master/examples/Marks/HeatMap.ipynb) along with the ipywidgets Range Slider to help us detect and clean outliers in our data. The class accepts a DataFrame and allows you to visually and programmatically filter your outliers. The cleaned DataFrame can then be retrieved through a simple convenience function."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from bqplot import (\n",
+    "    DateScale,\n",
+    "    ColorScale,\n",
+    "    HeatMap,\n",
+    "    Figure,\n",
+    "    LinearScale,\n",
+    "    OrdinalScale,\n",
+    "    Axis,\n",
+    ")\n",
+    "from scipy.stats import percentileofscore\n",
+    "from scipy.interpolate import interp1d\n",
+    "import bqplot.pyplot as plt\n",
+    "from traitlets import List, Float, observe\n",
+    "from ipywidgets import IntRangeSlider, Layout, VBox, HBox, jslink\n",
+    "from pandas import DatetimeIndex\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "\n",
+    "def quantile_space(x, q1=0.1, q2=0.9):\n",
+    "    \"\"\"\n",
+    "    Returns a function that squashes quantiles between q1 and q2\n",
+    "    \"\"\"\n",
+    "    q1_x, q2_x = np.percentile(x, [q1, q2])\n",
+    "    qs = np.percentile(x, np.linspace(0, 100, 100))\n",
+    "\n",
+    "    def get_quantile(t):\n",
+    "        return np.interp(t, qs, np.linspace(0, 100, 100))\n",
+    "\n",
+    "    def f(y):\n",
+    "        return np.interp(get_quantile(y), [0, q1, q2, 100], [-1, 0, 0, 1])\n",
+    "\n",
+    "    return f\n",
+    "\n",
+    "\n",
+    "class DNA(VBox):\n",
+    "\n",
+    "    colors = List()\n",
+    "    q1 = Float()\n",
+    "    q2 = Float()\n",
+    "\n",
+    "    def __init__(self, data, **kwargs):\n",
+    "        self.data = data\n",
+    "        date_x, date_y = False, False\n",
+    "        transpose = kwargs.pop(\"transpose\", False)\n",
+    "        if transpose is True:\n",
+    "            if type(data.index) is DatetimeIndex:\n",
+    "                self.x_scale = DateScale()\n",
+    "            if type(data.columns) is DatetimeIndex:\n",
+    "                self.y_scale = DateScale()\n",
+    "            x, y = list(data.columns.values), data.index.values\n",
+    "        else:\n",
+    "            if type(data.index) is DatetimeIndex:\n",
+    "                date_x = True\n",
+    "            if type(data.columns) is DatetimeIndex:\n",
+    "                date_y = True\n",
+    "            x, y = data.index.values, list(data.columns.values)\n",
+    "\n",
+    "        self.q1, self.q2 = kwargs.pop(\"quantiles\", (1, 99))\n",
+    "\n",
+    "        self.quant_func = quantile_space(\n",
+    "            self.data.values.flatten(), q1=self.q1, q2=self.q2\n",
+    "        )\n",
+    "        self.colors = kwargs.pop(\"colors\", [\"Red\", \"Black\", \"Green\"])\n",
+    "\n",
+    "        self.x_scale = DateScale() if date_x is True else LinearScale()\n",
+    "        self.y_scale = DateScale() if date_y is True else OrdinalScale(padding_y=0)\n",
+    "        self.color_scale = ColorScale(colors=self.colors)\n",
+    "        self.heat_map = HeatMap(\n",
+    "            color=self.quant_func(self.data.T),\n",
+    "            x=x,\n",
+    "            y=y,\n",
+    "            scales={\"x\": self.x_scale, \"y\": self.y_scale, \"color\": self.color_scale},\n",
+    "        )\n",
+    "        self.x_ax = Axis(scale=self.x_scale)\n",
+    "        self.y_ax = Axis(scale=self.y_scale, orientation=\"vertical\")\n",
+    "        show_axes = kwargs.pop(\"show_axes\", True)\n",
+    "        self.axes = [self.x_ax, self.y_ax] if show_axes is True else []\n",
+    "\n",
+    "        self.height = kwargs.pop(\"height\", \"800px\")\n",
+    "        self.layout = kwargs.pop(\n",
+    "            \"layout\", Layout(width=\"100%\", height=self.height, flex=\"1\")\n",
+    "        )\n",
+    "        self.fig_margin = kwargs.pop(\n",
+    "            \"fig_margin\", {\"top\": 60, \"bottom\": 60, \"left\": 150, \"right\": 0}\n",
+    "        )\n",
+    "        kwargs.setdefault(\"padding_y\", 0.0)\n",
+    "\n",
+    "        self.create_interaction(**kwargs)\n",
+    "\n",
+    "        self.figure = Figure(\n",
+    "            marks=[self.heat_map],\n",
+    "            axes=self.axes,\n",
+    "            fig_margin=self.fig_margin,\n",
+    "            layout=self.layout,\n",
+    "            min_aspect_ratio=0.0,\n",
+    "            **kwargs\n",
+    "        )\n",
+    "\n",
+    "        super(VBox, self).__init__(\n",
+    "            children=[self.range_slider, self.figure],\n",
+    "            layout=Layout(align_items=\"center\", width=\"100%\", height=\"100%\"),\n",
+    "            **kwargs\n",
+    "        )\n",
+    "\n",
+    "    def create_interaction(self, **kwargs):\n",
+    "        self.range_slider = IntRangeSlider(\n",
+    "            description=\"Filter Range\",\n",
+    "            value=(self.q1, self.q2),\n",
+    "            layout=Layout(width=\"100%\"),\n",
+    "        )\n",
+    "        self.range_slider.observe(self.slid_changed, \"value\")\n",
+    "        self.observe(self.changed, [\"q1\", \"q2\"])\n",
+    "\n",
+    "    def slid_changed(self, new):\n",
+    "        self.q1 = self.range_slider.value[0]\n",
+    "        self.q2 = self.range_slider.value[1]\n",
+    "\n",
+    "    def changed(self, new):\n",
+    "        self.range_slider.value = (self.q1, self.q2)\n",
+    "\n",
+    "        self.quant_func = quantile_space(\n",
+    "            self.data.values.flatten(), q1=self.q1, q2=self.q2\n",
+    "        )\n",
+    "        self.heat_map.color = self.quant_func(self.data.T)\n",
+    "\n",
+    "    def get_filtered_df(self, fill_type=\"median\"):\n",
+    "        q1_x, q2_x = np.percentile(self.data, [self.q1, self.q2])\n",
+    "        if fill_type == \"median\":\n",
+    "            return self.data[(self.data >= q1_x) & (self.data <= q2_x)].apply(\n",
+    "                lambda x: x.fillna(x.median())\n",
+    "            )\n",
+    "        elif fill_type == \"mean\":\n",
+    "            return self.data[(self.data >= q1_x) & (self.data <= q2_x)].apply(\n",
+    "                lambda x: x.fillna(x.mean())\n",
+    "            )\n",
+    "        else:\n",
+    "            raise ValueError(\"fill_type must be one of ('median', 'mean')\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We define the size of our matrix here. Larger matrices require a larger height."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "size = 100"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def num_to_col_letters(num):\n",
+    "    letters = \"\"\n",
+    "    while num:\n",
+    "        mod = (num - 1) % 26\n",
+    "        letters += chr(mod + 65)\n",
+    "        num = (num - 1) // 26\n",
+    "    return \"\".join(reversed(letters))\n",
+    "\n",
+    "\n",
+    "letters = []\n",
+    "\n",
+    "for i in range(1, size + 1):\n",
+    "    letters.append(num_to_col_letters(i))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = pd.DataFrame(np.random.randn(size, size), columns=letters)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_dna = DNA(\n",
+    "    data, title=\"DNA of our Data\", height=\"1400px\", colors=[\"Red\", \"White\", \"Green\"]\n",
+    ")\n",
+    "data_dna"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Instead of setting the quantiles by the sliders, we can also set them programmatically. Using a range of (5, 95) restricts the data considerably."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_dna.q1, data_dna.q2 = 5, 95"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Now, we can use the convenience function to extract a clean DataFrame."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "data_clean = data_dna.get_filtered_df()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The DNA fills outliers with the mean of the column. Alternately, we can fill the outliers by the mean."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_mean = data_dna.get_filtered_df(fill_type=\"mean\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can also visualize the new DataFrame the same way to test how our outliers look now."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DNA(data_clean, title=\"Cleaned Data\", height=\"1200px\", colors=[\"Red\", \"White\", \"Green\"])"
+   ]
+  }
+ ],
+ "metadata": {
+  "anaconda-cloud": {},
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

data/lib/starter_web/pages/public/_jupyter/notebooks/martin_renou_bqplot/Applications/Wealth Of Nations/Bubble Chart.ipynb

@@ -0,0 +1,255 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This is a `bqplot` recreation of Mike Bostock's [Wealth of Nations](https://bost.ocks.org/mike/nations/). This was also done by [Gapminder](http://www.gapminder.org/world/#$majorMode=chart$is;shi=t;ly=2003;lb=f;il=t;fs=11;al=30;stl=t;st=t;nsl=t;se=t$wst;tts=C$ts;sp=5.59290322580644;ti=2013$zpv;v=0$inc_x;mmid=XCOORDS;iid=phAwcNAVuyj1jiMAkmq1iMg;by=ind$inc_y;mmid=YCOORDS;iid=phAwcNAVuyj2tPLxKvvnNPA;by=ind$inc_s;uniValue=8.21;iid=phAwcNAVuyj0XOoBL_n5tAQ;by=ind$inc_c;uniValue=255;gid=CATID0;by=grp$map_x;scale=log;dataMin=194;dataMax=96846$map_y;scale=lin;dataMin=23;dataMax=86$map_s;sma=49;smi=2.65$cd;bd=0$inds=;modified=60). It is originally based on a TED Talk by [Hans Rosling](http://www.ted.com/talks/hans_rosling_shows_the_best_stats_you_ve_ever_seen)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "import ipywidgets as widgets\n",
+    "import bqplot as bq\n",
+    "import bqplot.pyplot as plt"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "initial_year = 1800"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "data = pd.read_json(\"nations.json\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def clean_data(data):\n",
+    "    for column in [\"income\", \"lifeExpectancy\", \"population\"]:\n",
+    "        data = data.drop(data[data[column].apply(len) <= 4].index)\n",
+    "    return data\n",
+    "\n",
+    "\n",
+    "def extrap_interp(data):\n",
+    "    data = np.array(data)\n",
+    "    x_range = np.arange(1800, 2009, 1.0)\n",
+    "    y_range = np.interp(x_range, data[:, 0], data[:, 1])\n",
+    "    return y_range\n",
+    "\n",
+    "\n",
+    "def extrap_data(data):\n",
+    "    for column in [\"income\", \"lifeExpectancy\", \"population\"]:\n",
+    "        data[column] = data[column].apply(extrap_interp)\n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = clean_data(data)\n",
+    "data = extrap_data(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "income_min, income_max = np.min(data[\"income\"].apply(np.min)), np.max(\n",
+    "    data[\"income\"].apply(np.max)\n",
+    ")\n",
+    "life_exp_min, life_exp_max = np.min(data[\"lifeExpectancy\"].apply(np.min)), np.max(\n",
+    "    data[\"lifeExpectancy\"].apply(np.max)\n",
+    ")\n",
+    "pop_min, pop_max = np.min(data[\"population\"].apply(np.min)), np.max(\n",
+    "    data[\"population\"].apply(np.max)\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_data(year):\n",
+    "    year_index = year - 1800\n",
+    "    income = data[\"income\"].apply(lambda x: x[year_index])\n",
+    "    life_exp = data[\"lifeExpectancy\"].apply(lambda x: x[year_index])\n",
+    "    pop = data[\"population\"].apply(lambda x: x[year_index])\n",
+    "    return income, life_exp, pop"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "time_interval = 100\n",
+    "fig_layout = widgets.Layout(width=\"1000px\", height=\"700px\", overflow_x=\"hidden\")\n",
+    "fig = plt.figure(\n",
+    "    layout=fig_layout,\n",
+    "    fig_margin=dict(top=60, bottom=80, left=40, right=20),\n",
+    "    title=\"Health and Wealth of Nations\",\n",
+    "    animation_duration=time_interval,\n",
+    ")\n",
+    "\n",
+    "plt.scales(\n",
+    "    scales={\n",
+    "        \"x\": bq.LogScale(min=min(200, income_min), max=income_max),\n",
+    "        \"y\": bq.LinearScale(min=life_exp_min, max=life_exp_max),\n",
+    "        \"color\": bq.OrdinalColorScale(\n",
+    "            domain=data[\"region\"].unique().tolist(), colors=bq.CATEGORY10[:6]\n",
+    "        ),\n",
+    "        \"size\": bq.LinearScale(min=pop_min, max=pop_max),\n",
+    "    }\n",
+    ")\n",
+    "\n",
+    "# add custom x tick values\n",
+    "ticks = [2, 4, 6, 8, 10]\n",
+    "income_ticks = (\n",
+    "    [t * 100 for t in ticks] + [t * 1000 for t in ticks] + [t * 10000 for t in ticks]\n",
+    ")\n",
+    "\n",
+    "# custom axis options\n",
+    "axes_options = {\n",
+    "    \"x\": dict(\n",
+    "        label=\"Income per Capita\",\n",
+    "        label_location=\"end\",\n",
+    "        label_offset=\"-2ex\",\n",
+    "        tick_format=\"~s\",\n",
+    "        tick_values=income_ticks,\n",
+    "    ),\n",
+    "    \"y\": dict(\n",
+    "        label=\"Life Expectancy\",\n",
+    "        orientation=\"vertical\",\n",
+    "        side=\"left\",\n",
+    "        label_location=\"end\",\n",
+    "        label_offset=\"-1em\",\n",
+    "    ),\n",
+    "    \"color\": dict(label=\"Region\"),\n",
+    "}\n",
+    "\n",
+    "tooltip = bq.Tooltip(\n",
+    "    fields=[\"name\", \"x\", \"y\"],\n",
+    "    labels=[\"Country Name\", \"Income per Capita\", \"Life Expectancy\"],\n",
+    ")\n",
+    "\n",
+    "year_label = bq.Label(\n",
+    "    x=[0.75],\n",
+    "    y=[0.10],\n",
+    "    default_size=46,\n",
+    "    font_weight=\"bolder\",\n",
+    "    colors=[\"orange\"],\n",
+    "    text=[str(initial_year)],\n",
+    "    enable_move=True,\n",
+    ")\n",
+    "\n",
+    "# Start with the first year's data\n",
+    "cap_income, life_exp, pop = get_data(initial_year)\n",
+    "\n",
+    "wealth_scat = plt.scatter(\n",
+    "    cap_income,\n",
+    "    life_exp,\n",
+    "    color=data[\"region\"],\n",
+    "    size=pop,\n",
+    "    names=data[\"name\"],\n",
+    "    display_names=False,\n",
+    "    default_size=20000,\n",
+    "    tooltip=tooltip,\n",
+    "    stroke=\"Black\",\n",
+    "    axes_options=axes_options,\n",
+    "    unhovered_style={\"opacity\": 0.5},\n",
+    ")\n",
+    "nation_line = plt.plot(\n",
+    "    data[\"income\"][0], data[\"lifeExpectancy\"][0], colors=[\"Gray\"], visible=False\n",
+    ")\n",
+    "\n",
+    "# slider for the year\n",
+    "year_slider = widgets.IntSlider(\n",
+    "    min=1800, max=2008, description=\"Year\", value=initial_year\n",
+    ")\n",
+    "\n",
+    "# register callbacks\n",
+    "def hover_changed(change):\n",
+    "    if change.new is not None:\n",
+    "        nation_line.x = data[data[\"name\"] == wealth_scat.names[change.new]][\n",
+    "            \"income\"\n",
+    "        ].values[0]\n",
+    "        nation_line.y = data[data[\"name\"] == wealth_scat.names[change.new]][\n",
+    "            \"lifeExpectancy\"\n",
+    "        ].values[0]\n",
+    "        nation_line.visible = True\n",
+    "    else:\n",
+    "        nation_line.visible = False\n",
+    "\n",
+    "\n",
+    "wealth_scat.observe(hover_changed, \"hovered_point\")\n",
+    "\n",
+    "\n",
+    "def year_changed(change):\n",
+    "    wealth_scat.x, wealth_scat.y, wealth_scat.size = get_data(year_slider.value)\n",
+    "    year_label.text = [str(year_slider.value)]\n",
+    "\n",
+    "\n",
+    "year_slider.observe(year_changed, \"value\")\n",
+    "\n",
+    "play_button = widgets.Play(min=1800, max=2008, interval=time_interval)\n",
+    "widgets.jslink((play_button, \"value\"), (year_slider, \"value\"))\n",
+    "\n",
+    "widgets.VBox([widgets.HBox([play_button, year_slider]), fig])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}